From 2ded7feaffa48da289ae7e4c0c79a8b0ab1e5cd2 Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Tue, 23 Jan 2024 17:08:12 +0200 Subject: [PATCH 01/32] started rewriting bound_tightening Also trained a paraboloid Flux NN model --- Project.toml | 2 + README.md | 13 +- src/neural_networks/NN_paraboloid.bson | Bin 0 -> 4668 bytes .../bound_tightening_broken.jl | 677 +++++++++++++++++ src/neural_networks/bound_tightening_new.jl | 688 +----------------- ...{TE_training.jl => paraboloid_training.jl} | 0 6 files changed, 717 insertions(+), 663 deletions(-) create mode 100644 src/neural_networks/NN_paraboloid.bson create mode 100644 src/neural_networks/bound_tightening_broken.jl rename test/tree_ensembles/{TE_training.jl => paraboloid_training.jl} (100%) diff --git a/Project.toml b/Project.toml index a5f23d8..d435756 100644 --- a/Project.toml +++ b/Project.toml @@ -4,6 +4,7 @@ authors = ["Nikita Belyak and contributors"] version = "1.0.0-DEV" [deps] +BSON = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" EvoTrees = "f6006082-12f8-11e9-0c9c-0d5d367ab1e5" Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" @@ -13,6 +14,7 @@ JuMP = "4076af6c-e467-56ae-b986-b466b2749572" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" MLDatasets = "eb30cadb-4394-5ae3-aed4-317e484a6458" NPZ = "15e1cf62-19b3-5cfa-8e77-841668bca605" +Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SCIP = "82193955-e24f-5292-bf16-6f2c5261a85f" SharedArrays = "1a1011a3-84de-559e-8e89-a11a2f7dc383" diff --git a/README.md b/README.md index 9acd97e..a7889d4 100644 --- a/README.md +++ b/README.md @@ -13,10 +13,6 @@ Currently supported models include neural networks using ReLU activation and tre * **tree ensemble to MIP conversion** - obtain an integer optimization problem from a trained tree ensemble model * **tree ensemble optimization** - optimize a trained decision tree model, i.e., find an input that maximizes the forest output -#### Interface -1. *extract_evotrees_info* - obtain a universal datatype TEModel from a trained EvoTrees model -2. *tree_model_to_MIP* - obtain the integer programming JuMP model (with or without the split constraints) - #### Workflow trained tree ensemble model $\rightarrow$ TEModel $\rightarrow$ JuMP model $\rightarrow$ optimization @@ -27,12 +23,5 @@ trained tree ensemble model $\rightarrow$ TEModel $\rightarrow$ JuMP model $\rig * **neural network compression** - reduce network size by removing unnecessary nodes * **neural network optimization** - find the input that maximizes the neural network output -#### Interface -1. *create_JuMP_model* -2. *create_CNN_JuMP_model* -3. *bound_tightening* -4. *evaluate!* -5. *compress_network* - #### Workflow -trained neural network $\rightarrow$ NNModel $\rightarrow$ JuMP model $\rightarrow$ bound tightening $\rightarrow$ compression $\rightarrow$ optimization \ No newline at end of file +trained Flux NN model $\rightarrow$ JuMP model $\rightarrow$ bound tightening $\rightarrow$ compression $\rightarrow$ optimization \ No newline at end of file diff --git a/src/neural_networks/NN_paraboloid.bson b/src/neural_networks/NN_paraboloid.bson new file mode 100644 index 0000000000000000000000000000000000000000..313a893a8fd971bab1168a617a533350d5a670a7 GIT binary patch literal 4668 zcmcB!Vqjp-%}+_qVK~Iiz`#_Jn9jh?z`#&kQdF8;!oXZoS&+(L!wi+-1j?i&mL!5j zSqc)15_5|gelh`#Fktuyq`=y-Dq06o^Z;l8Zbfr}vOtqO@hV~k$}+IzCFZ6w$b%Fb zFt7nBKfj#JBnBo!Fh{v4HK!CP$ZW`<3{nE}6bF!zl$e}dl$ypM1R_Ar1OsLxJgT^0 zs-RZN0r@~HL5^_CDXoCn=#rXOoC*{J+V}vZ0%RlBa2N(&G#d6^#Efg_{GwD?T!)kv z{uWLP8*;kTLLx*BJT{5KMc!57+LTEP{VLe zRghpf#0U0|+*MMzF&fVdMR%Aj&ZZeP4i`o6*&JM6Y7?Agr#EZOp5 z?!rh>R4Zj6Q3noupk=_YI6uNu@i|B;J^%}PI;CP@B@G6^R18bd(9~f75)a@rviEv@ zY@g(UP5To1z4z_Cd3JB^Lt*=-xcmF`6Xo}rHahNKJ4?WRxyp@wPX8Uisd^$%A3akw z%nML=^MKs_Z_`~{5PkP~jNQPbXd{NjBRoYff~4p (verbose ? 1 : 0), "TimeLimit" => tl)) + + # keeps track of the current node index starting from layer 1 (out of 0:K) + outer_index = node_count[1] + 1 + + # NOTE! below variables and constraints for all opt problems + @variable(model, x[k in 0:K, j in 1:node_count[k+1]] >= 0) + @variable(model, s[k in 1:K-1, j in 1:node_count[k+1]] >= 0) + @variable(model, z[k in 1:K-1, j in 1:node_count[k+1]], Bin) + @variable(model, U[k in 0:K, j in 1:node_count[k+1]]) + @variable(model, L[k in 0:K, j in 1:node_count[k+1]]) + + # fix values to all U[k,j] and L[k,j] from U_bounds and L_bounds + index = 1 + for k in 0:K + for j in 1:node_count[k+1] + fix(U[k, j], curr_U_bounds[index], force=true) + fix(L[k, j], curr_L_bounds[index], force=true) + index += 1 + end + end + + # input layer (layer 0) node bounds are given beforehand + for input_node in 1:node_count[1] + delete_lower_bound(x[0, input_node]) + @constraint(model, L[0, input_node] <= x[0, input_node]) + @constraint(model, x[0, input_node] <= U[0, input_node]) + end + + # deleting lower bound for output nodes + for output_node in 1:node_count[K+1] + delete_lower_bound(x[K, output_node]) + end + + # NOTE! below constraints depending on the layer + for k in 1:K + # we only want to build ALL of the constraints until the PREVIOUS layer, and then go node by node + # here we calculate ONLY the constraints until the PREVIOUS layer + for node_in in 1:node_count[k] + if k >= 2 + temp_sum = sum(W[k-1][node_in, j] * x[k-1-1, j] for j in 1:node_count[k-1]) + @constraint(model, x[k-1, node_in] <= U[k-1, node_in] * z[k-1, node_in]) + @constraint(model, s[k-1, node_in] <= -L[k-1, node_in] * (1 - z[k-1, node_in])) + if k <= K - 1 + @constraint(model, temp_sum + b[k-1][node_in] == x[k-1, node_in] - s[k-1, node_in]) + else # k == K + @constraint(model, temp_sum + b[k-1][node_in] == x[k-1, node_in]) + end + end + end + + # NOTE! below constraints depending on the node + for node in 1:node_count[k+1] + # here we calculate the specific constraints depending on the current node + temp_sum = sum(W[k][node, j] * x[k-1, j] for j in 1:node_count[k]) # NOTE! prev layer [k] + if k <= K - 1 + @constraint(model, node_con, temp_sum + b[k][node] == x[k, node] - s[k, node]) + @constraint(model, node_U, x[k, node] <= U[k, node] * z[k, node]) + @constraint(model, node_L, s[k, node] <= -L[k, node] * (1 - z[k, node])) + elseif k == K # == last value of k + @constraint(model, node_con, temp_sum + b[k][node] == x[k, node]) + @constraint(model, node_L, L[k, node] <= x[k, node]) + @constraint(model, node_U, x[k, node] <= U[k, node]) + end + + # NOTE! below objective function and optimizing the model depending on obj_function and layer + for obj_function in 1:2 + if obj_function == 1 && k <= K - 1 # Min, hidden layer + @objective(model, Min, x[k, node] - s[k, node]) + elseif obj_function == 2 && k <= K - 1 # Max, hidden layer + @objective(model, Max, x[k, node] - s[k, node]) + elseif obj_function == 1 && k == K # Min, last layer + @objective(model, Min, x[k, node]) + elseif obj_function == 2 && k == K # Max, last layer + @objective(model, Max, x[k, node]) + end + + solve_time = @elapsed optimize!(model) + solve_time = round(solve_time; sigdigits = 3) + # @assert termination_status(model) == OPTIMAL || termination_status(model) == TIME_LIMIT + # "Problem (layer $k (from 1:$K), node $node, $(obj_function == 1 ? "L" : "U")-bound) is infeasible." + optimal = objective_value(model) + println("Layer $k, node $node, $(obj_function == 1 ? "L" : "U")-bound: solve time $(solve_time)s, optimal value $(optimal)") + + # fix the model variable L or U corresponding to the current node to be the optimal value + if obj_function == 1 # Min + curr_L_bounds[outer_index] = optimal + fix(L[k, node], optimal) + elseif obj_function == 2 # Max + curr_U_bounds[outer_index] = optimal + fix(U[k, node], optimal) + end + end + outer_index += 1 + + # deleting and unregistering the constraints assigned to the current node + delete(model, node_con) + delete(model, node_L) + delete(model, node_U) + unregister(model, :node_con) + unregister(model, :node_L) + unregister(model, :node_U) + end + end + + println("Solving optimal constraint bounds single-threaded complete") + + return curr_U_bounds, curr_L_bounds +end + +""" +bound_tightening_threads(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false, tl::float64=1) + +A multi-threaded (using Threads) implementation of optimal tightened constraint bounds L and U for for a trained DNN. +Using these bounds with the create_JuMP_model function reduces solution time for optimization problems. + +# Arguments +- `DNN::Chain`: A trained ReLU DNN. +- `init_U_bounds::Vector{Float32}`: Initial upper bounds on the node values of the DNN. +- `init_L_bounds::Vector{Float32}`: Initial lower bounds on the node values of the DNN. +- `verbose::Bool=false`: Controls Gurobi logs. +- `tl::Float64=1.0`: Controls the time limit for solvign the subproblems + +# Examples +```julia +L_bounds_threads, U_bounds_threads = bound_tightening_threads(DNN, init_U_bounds, init_L_bounds, false, 1.0) +``` +""" +function bound_tightening_threads(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false, tl::Float64=1.0) + + K = length(DNN) # NOTE! there are K+1 layers in the nn + + # store the DNN weights and biases + DNN_params = params(DNN) + W = [DNN_params[2*i-1] for i in 1:K] + b = [DNN_params[2*i] for i in 1:K] + + # stores the node count of layer k (starting at layer k=0) at index k+1 + input_node_count = length(DNN_params[1][1, :]) + node_count = [if k == 1 input_node_count else length(DNN_params[2*(k-1)]) end for k in 1:K+1] + + # store the current optimal bounds in the algorithm + curr_U_bounds = copy(init_U_bounds) + curr_L_bounds = copy(init_L_bounds) + + lock = Threads.ReentrantLock() + + for k in 1:K + + Threads.@threads for node in 1:(2*node_count[k+1]) # loop over both obj functions + + ### below variables and constraints in all problems + + model = Model(optimizer_with_attributes(Gurobi.Optimizer, "OutputFlag" => (verbose ? 1 : 0), "TimeLimit" => tl)) + + # keeps track of the current node index starting from layer 1 (out of 0:K) + prev_layers_node_sum = 0 + for prev_layer in 0:k-1 + prev_layers_node_sum += node_count[prev_layer+1] + end + + # loops nodes twice: 1st time with obj function Min, 2nd time with Max + curr_node = node + obj_function = 1 + if node > node_count[k+1] + curr_node = node - node_count[k+1] + obj_function = 2 + end + curr_node_index = prev_layers_node_sum + curr_node + + # NOTE! below variables and constraints for all opt problems + @variable(model, x[k in 0:K, j in 1:node_count[k+1]] >= 0) + @variable(model, s[k in 1:K-1, j in 1:node_count[k+1]] >= 0) + @variable(model, z[k in 1:K-1, j in 1:node_count[k+1]], Bin) + @variable(model, U[k in 0:K, j in 1:node_count[k+1]]) + @variable(model, L[k in 0:K, j in 1:node_count[k+1]]) + + # fix values to all U[k,j] and L[k,j] from U_bounds and L_bounds + index = 1 + Threads.lock(lock) do + for k in 0:K + for j in 1:node_count[k+1] + fix(U[k, j], curr_U_bounds[index], force=true) + fix(L[k, j], curr_L_bounds[index], force=true) + index += 1 + end + end + end + + # input layer (layer 0) node bounds are given beforehand + for input_node in 1:node_count[1] + delete_lower_bound(x[0, input_node]) + @constraint(model, L[0, input_node] <= x[0, input_node]) + @constraint(model, x[0, input_node] <= U[0, input_node]) + end + + # deleting lower bound for output nodes + for output_node in 1:node_count[K+1] + delete_lower_bound(x[K, output_node]) + end + + ### below constraints depending on the layer (every constraint up to the previous layer) + for k_in in 1:k + for node_in in 1:node_count[k_in] + if k_in >= 2 + temp_sum = sum(W[k_in-1][node_in, j] * x[k_in-1-1, j] for j in 1:node_count[k_in-1]) + @constraint(model, x[k_in-1, node_in] <= U[k_in-1, node_in] * z[k_in-1, node_in]) + @constraint(model, s[k_in-1, node_in] <= -L[k_in-1, node_in] * (1 - z[k_in-1, node_in])) + if k_in <= K - 1 + @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in] - s[k_in-1, node_in]) + else # k_in == K + @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in]) + end + end + end + end + + ### below constraints depending on the node + temp_sum = sum(W[k][curr_node, j] * x[k-1, j] for j in 1:node_count[k]) # NOTE! prev layer [k] + if k <= K - 1 + @constraint(model, node_con, temp_sum + b[k][curr_node] == x[k, curr_node] - s[k, curr_node]) + @constraint(model, node_U, x[k, curr_node] <= U[k, curr_node] * z[k, curr_node]) + @constraint(model, node_L, s[k, curr_node] <= -L[k, curr_node] * (1 - z[k, curr_node])) + elseif k == K # == last value of k + @constraint(model, node_con, temp_sum + b[k][curr_node] == x[k, curr_node]) + @constraint(model, node_L, L[k, curr_node] <= x[k, curr_node]) + @constraint(model, node_U, x[k, curr_node] <= U[k, curr_node]) + end + + if obj_function == 1 && k <= K - 1 # Min, hidden layer + @objective(model, Min, x[k, curr_node] - s[k, curr_node]) + elseif obj_function == 2 && k <= K - 1 # Max, hidden layer + @objective(model, Max, x[k, curr_node] - s[k, curr_node]) + elseif obj_function == 1 && k == K # Min, last layer + @objective(model, Min, x[k, curr_node]) + elseif obj_function == 2 && k == K # Max, last layer + @objective(model, Max, x[k, curr_node]) + end + + solve_time = @elapsed optimize!(model) + solve_time = round(solve_time; sigdigits = 3) + # @assert termination_status(model) == OPTIMAL || termination_status(model) == TIME_LIMIT + # "Problem (layer $k (from 1:$K), node $curr_node, $(obj_function == 1 ? "L" : "U")-bound) is infeasible." + optimal = objective_value(model) + + # @show termination_status(model) + # if termination_status(model) == OPTIMAL + # optimal = objective_value(model) + # else + # optimal = Inf + # end + + + println("Thread: $(Threads.threadid()), layer $k, node $curr_node, $(obj_function == 1 ? "L" : "U")-bound: solve time $(solve_time)s, optimal value $(optimal)") + + # fix the model variable L or U corresponding to the current node to be the optimal value + Threads.lock(lock) do + if obj_function == 1 && optimal != Inf # Min and we recieved a new bound + + curr_L_bounds[curr_node_index] = optimal + fix(L[k, curr_node], optimal) + + elseif obj_function == 2 && optimal != Inf # Max and we recieved a new bound + + curr_U_bounds[curr_node_index] = optimal + fix(U[k, curr_node], optimal) + + end + end + + end + + end + + println("Solving optimal constraint bounds using threads complete") + + return curr_U_bounds, curr_L_bounds +end + +""" +bound_tightening_workers(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false, tl::Float64=1.0) + +A multi-threaded (using workers) implementation of optimal tightened constraint bounds L and U for for a trained DNN. +Using these bounds with the create_JuMP_model function reduces solution time for optimization problems. + +# Arguments +- `DNN::Chain`: A trained ReLU DNN. +- `init_U_bounds::Vector{Float32}`: Initial upper bounds on the node values of the DNN. +- `init_L_bounds::Vector{Float32}`: Initial lower bounds on the node values of the DNN. +- `verbose::Bool=false`: Controls Gurobi logs. +- `tl::Float64=1.0`: Controls the time limit for solvign the subproblems + +# Examples +```julia +L_bounds_workers, U_bounds_workers = bound_tightening_workers(DNN, init_U_bounds, init_L_bounds, false, 1.0) +``` +""" +function bound_tightening_workers(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false, tl::Float64=1.0) + + K = length(DNN) # NOTE! there are K+1 layers in the nn + + # store the DNN weights and biases + DNN_params = params(DNN) + W = [DNN_params[2*i-1] for i in 1:K] + b = [DNN_params[2*i] for i in 1:K] + + # stores the node count of layer k (starting at layer k=0) at index k+1 + input_node_count = length(DNN_params[1][1, :]) + node_count = [if k == 1 input_node_count else length(DNN_params[2*(k-1)]) end for k in 1:K+1] + + # store the current optimal bounds in the algorithm + curr_U_bounds = copy(init_U_bounds) + curr_L_bounds = copy(init_L_bounds) + + for k in 1:K + + # Distributed.pmap returns the bounds in order + L_U_bounds = Distributed.pmap(node -> bt_workers_inner(K, k, node, W, b, node_count, curr_U_bounds, curr_L_bounds, verbose, tl), 1:(2*node_count[k+1])) + + for node in 1:node_count[k+1] + prev_layers_node_sum = 0 + for prev_layer in 0:k-1 + prev_layers_node_sum += node_count[prev_layer+1] + end + + # loops nodes twice: 1st time with obj function Min, 2nd time with Max + curr_node = node + obj_function = 1 + if node > node_count[k+1] + curr_node = node - node_count[k+1] + obj_function = 2 + end + curr_node_index = prev_layers_node_sum + curr_node + + # L-bounds in 1:node_count[k+1], U-bounds in 1:(node + node_count[k+1]) + curr_L_bounds[curr_node_index] = L_U_bounds[node] + curr_U_bounds[curr_node_index] = L_U_bounds[node + node_count[k+1]] + end + + end + + println("Solving optimal constraint bounds using workers complete") + + return curr_U_bounds, curr_L_bounds +end + +# Inner function to bound_tightening_workers: assigns a JuMP model to the current worker + +function bt_workers_inner( + K::Int64, + k::Int64, + node::Int64, + W::Vector{Matrix{Float32}}, + b::Vector{Vector{Float32}}, + node_count::Vector{Int64}, + curr_U_bounds::Vector{Float32}, + curr_L_bounds::Vector{Float32}, + verbose::Bool, + tl::Float64 + ) + + model = Model(optimizer_with_attributes(Gurobi.Optimizer, "OutputFlag" => (verbose ? 1 : 0), "Threads" => 1, "TimeLimit" => tl)) + + # keeps track of the current node index starting from layer 1 (out of 0:K) + prev_layers_node_sum = 0 + for prev_layer in 0:k-1 + prev_layers_node_sum += node_count[prev_layer+1] + end + + # loops nodes twice: 1st time with obj function Min, 2nd time with Max + curr_node = node + obj_function = 1 + if node > node_count[k+1] + curr_node = node - node_count[k+1] + obj_function = 2 + end + + # NOTE! below variables and constraints for all opt problems + @variable(model, x[k in 0:K, j in 1:node_count[k+1]] >= 0) + @variable(model, s[k in 1:K-1, j in 1:node_count[k+1]] >= 0) + @variable(model, z[k in 1:K-1, j in 1:node_count[k+1]], Bin) + @variable(model, U[k in 0:K, j in 1:node_count[k+1]]) + @variable(model, L[k in 0:K, j in 1:node_count[k+1]]) + + # fix values to all U[k,j] and L[k,j] from U_bounds and L_bounds + index = 1 + for k in 0:K + for j in 1:node_count[k+1] + fix(U[k, j], curr_U_bounds[index], force=true) + fix(L[k, j], curr_L_bounds[index], force=true) + index += 1 + end + end + + # input layer (layer 0) node bounds are given beforehand + for input_node in 1:node_count[1] + delete_lower_bound(x[0, input_node]) + @constraint(model, L[0, input_node] <= x[0, input_node]) + @constraint(model, x[0, input_node] <= U[0, input_node]) + end + + # deleting lower bound for output nodes + for output_node in 1:node_count[K+1] + delete_lower_bound(x[K, output_node]) + end + + ### below constraints depending on the layer (every constraint up to the previous layer) + for k_in in 1:k + for node_in in 1:node_count[k_in] + if k_in >= 2 + temp_sum = sum(W[k_in-1][node_in, j] * x[k_in-1-1, j] for j in 1:node_count[k_in-1]) + @constraint(model, x[k_in-1, node_in] <= U[k_in-1, node_in] * z[k_in-1, node_in]) + @constraint(model, s[k_in-1, node_in] <= -L[k_in-1, node_in] * (1 - z[k_in-1, node_in])) + if k_in <= K - 1 + @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in] - s[k_in-1, node_in]) + else # k_in == K + @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in]) + end + end + end + end + + ### below constraints depending on the node + temp_sum = sum(W[k][curr_node, j] * x[k-1, j] for j in 1:node_count[k]) # NOTE! prev layer [k] + if k <= K - 1 + @constraint(model, node_con, temp_sum + b[k][curr_node] == x[k, curr_node] - s[k, curr_node]) + @constraint(model, node_U, x[k, curr_node] <= U[k, curr_node] * z[k, curr_node]) + @constraint(model, node_L, s[k, curr_node] <= -L[k, curr_node] * (1 - z[k, curr_node])) + elseif k == K # == last value of k + @constraint(model, node_con, temp_sum + b[k][curr_node] == x[k, curr_node]) + @constraint(model, node_L, L[k, curr_node] <= x[k, curr_node]) + @constraint(model, node_U, x[k, curr_node] <= U[k, curr_node]) + end + + if obj_function == 1 && k <= K - 1 # Min, hidden layer + @objective(model, Min, x[k, curr_node] - s[k, curr_node]) + elseif obj_function == 2 && k <= K - 1 # Max, hidden layer + @objective(model, Max, x[k, curr_node] - s[k, curr_node]) + elseif obj_function == 1 && k == K # Min, last layer + @objective(model, Min, x[k, curr_node]) + elseif obj_function == 2 && k == K # Max, last layer + @objective(model, Max, x[k, curr_node]) + end + + solve_time = @elapsed optimize!(model) + solve_time = round(solve_time; sigdigits = 3) + # @assert termination_status(model) == OPTIMAL || termination_status(model) == TIME_LIMIT + # "Problem (layer $k (from 1:$K), node $curr_node, $(obj_function == 1 ? "L" : "U")-bound) is infeasible." + optimal = objective_value(model) + println("Worker: $(myid()), layer $k, node $curr_node, $(obj_function == 1 ? "L" : "U")-bound: solve time $(solve_time)s, optimal value $(optimal)") + + return optimal +end + +""" +bound_tightening_2workers(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false) + +A multi-threaded (using workers) implementation of optimal tightened constraint bounds L and U for for a trained DNN. +This function uses two in-place models at each layer to reduce memory usage. A max of 2 workers in use simultaneously. +Using these bounds with the create_JuMP_model function reduces solution time for optimization problems. + +# Arguments +- `DNN::Chain`: A trained ReLU DNN. +- `init_U_bounds::Vector{Float32}`: Initial upper bounds on the node values of the DNN. +- `init_L_bounds::Vector{Float32}`: Initial lower bounds on the node values of the DNN. +- `verbose::Bool=false`: Controls Gurobi logs. +- `tl::Float64=1.0`: Controls the time limit for solvign the subproblems. + +# Examples +```julia +L_bounds_workers, U_bounds_workers = bound_tightening_2workers(DNN, init_U_bounds, init_L_bounds, false, 1.0) +``` +""" +function bound_tightening_2workers(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false, tl::Float64=1.0) + + K = length(DNN) # NOTE! there are K+1 layers in the nn + + # store the DNN weights and biases + DNN_params = params(DNN) + W = [DNN_params[2*i-1] for i in 1:K] + b = [DNN_params[2*i] for i in 1:K] + + # stores the node count of layer k (starting at layer k=0) at index k+1 + input_node_count = length(DNN_params[1][1, :]) + node_count = [if k == 1 input_node_count else length(DNN_params[2*(k-1)]) end for k in 1:K+1] + + # store the current optimal bounds in the algorithm + curr_U_bounds = copy(init_U_bounds) + curr_L_bounds = copy(init_L_bounds) + + # split the available threads into 2 to be assigned to each worker (integer division) + n = Threads.nthreads() + threads_split = [n÷2, n-(n÷2)] + + for k in 1:K + + L_U_bounds = Distributed.pmap(obj_function -> + bt_2workers_inner(K, k, obj_function, W, b, node_count, curr_U_bounds, curr_L_bounds, threads_split[obj_function], verbose, tl), 1:2) + + curr_L_bounds = L_U_bounds[1] + curr_U_bounds = L_U_bounds[2] + + end + + println("Solving optimal constraint bounds complete") + + return curr_U_bounds, curr_L_bounds +end + + +# Inner function to solve_optimal_bounds_2workers: solves L or U bounds for all nodes in a layer using the same JuMP model + +function bt_2workers_inner( + K::Int64, + k::Int64, + obj_function::Int64, + W::Vector{Matrix{Float32}}, + b::Vector{Vector{Float32}}, + node_count::Vector{Int64}, + curr_U_bounds::Vector{Float32}, + curr_L_bounds::Vector{Float32}, + n_threads::Int64, + verbose::Bool, + tl::Float64 + ) + + curr_U_bounds_copy = copy(curr_U_bounds) + curr_L_bounds_copy = copy(curr_L_bounds) + + model = Model(optimizer_with_attributes(Gurobi.Optimizer, "OutputFlag" => (verbose ? 1 : 0), "Threads" => n_threads, "TimeLimit" => tl)) + + # NOTE! below variables and constraints for all opt problems + @variable(model, x[k in 0:K, j in 1:node_count[k+1]] >= 0) + @variable(model, s[k in 1:K-1, j in 1:node_count[k+1]] >= 0) + @variable(model, z[k in 1:K-1, j in 1:node_count[k+1]], Bin) + @variable(model, U[k in 0:K, j in 1:node_count[k+1]]) + @variable(model, L[k in 0:K, j in 1:node_count[k+1]]) + + # fix values to all U[k,j] and L[k,j] from U_bounds and L_bounds + index = 1 + for k in 0:K + for j in 1:node_count[k+1] + fix(U[k, j], curr_U_bounds[index], force=true) + fix(L[k, j], curr_L_bounds[index], force=true) + index += 1 + end + end + + # input layer (layer 0) node bounds are given beforehand + for input_node in 1:node_count[1] + delete_lower_bound(x[0, input_node]) + @constraint(model, L[0, input_node] <= x[0, input_node]) + @constraint(model, x[0, input_node] <= U[0, input_node]) + end + + # deleting lower bound for output nodes + for output_node in 1:node_count[K+1] + delete_lower_bound(x[K, output_node]) + end + + ### below constraints depending on the layer (every constraint up to the previous layer) + for k_in in 1:k + for node_in in 1:node_count[k_in] + if k_in >= 2 + temp_sum = sum(W[k_in-1][node_in, j] * x[k_in-1-1, j] for j in 1:node_count[k_in-1]) + @constraint(model, x[k_in-1, node_in] <= U[k_in-1, node_in] * z[k_in-1, node_in]) + @constraint(model, s[k_in-1, node_in] <= -L[k_in-1, node_in] * (1 - z[k_in-1, node_in])) + if k_in <= K - 1 + @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in] - s[k_in-1, node_in]) + else # k_in == K + @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in]) + end + end + end + end + + for node in 1:node_count[k+1] + + prev_layers_node_sum = 0 + for prev_layer in 0:k-1 + prev_layers_node_sum += node_count[prev_layer+1] + end + curr_node_index = prev_layers_node_sum + node + + ### below constraints depending on the node + temp_sum = sum(W[k][node, j] * x[k-1, j] for j in 1:node_count[k]) # NOTE! prev layer [k] + if k <= K - 1 + @constraint(model, node_con, temp_sum + b[k][node] == x[k, node] - s[k, node]) + @constraint(model, node_U, x[k, node] <= U[k, node] * z[k, node]) + @constraint(model, node_L, s[k, node] <= -L[k, node] * (1 - z[k, node])) + elseif k == K # == last value of k + @constraint(model, node_con, temp_sum + b[k][node] == x[k, node]) + @constraint(model, node_L, L[k, node] <= x[k, node]) + @constraint(model, node_U, x[k, node] <= U[k, node]) + end + + if obj_function == 1 && k <= K - 1 # Min, hidden layer + @objective(model, Min, x[k, node] - s[k, node]) + elseif obj_function == 2 && k <= K - 1 # Max, hidden layer + @objective(model, Max, x[k, node] - s[k, node]) + elseif obj_function == 1 && k == K # Min, last layer + @objective(model, Min, x[k, node]) + elseif obj_function == 2 && k == K # Max, last layer + @objective(model, Max, x[k, node]) + end + + solve_time = @elapsed optimize!(model) + solve_time = round(solve_time; sigdigits = 3) + @assert termination_status(model) == OPTIMAL || termination_status(model) == TIME_LIMIT + "Problem (layer $k (from 1:$K), node $node, $(obj_function == 1 ? "L" : "U")-bound) is infeasible." + optimal = objective_value(model) + println("Worker: $(myid()), layer $k, node $node, $(obj_function == 1 ? "L" : "U")-bound: solve time $(solve_time)s, optimal value $(optimal)") + + # fix the model variable L or U corresponding to the current node to be the optimal value + if obj_function == 1 # Min + curr_L_bounds_copy[curr_node_index] = optimal + elseif obj_function == 2 # Max + curr_U_bounds_copy[curr_node_index] = optimal + end + + # deleting and unregistering the constraints assigned to the current node + delete(model, node_con) + delete(model, node_L) + delete(model, node_U) + unregister(model, :node_con) + unregister(model, :node_L) + unregister(model, :node_U) + end + + if obj_function == 1 # Min + return curr_L_bounds_copy + elseif obj_function == 2 # Max + return curr_U_bounds_copy + end + +end \ No newline at end of file diff --git a/src/neural_networks/bound_tightening_new.jl b/src/neural_networks/bound_tightening_new.jl index ae4699b..b0dd1a0 100644 --- a/src/neural_networks/bound_tightening_new.jl +++ b/src/neural_networks/bound_tightening_new.jl @@ -1,677 +1,63 @@ -using JuMP, Flux, Gurobi -using JuMP: Model -using Flux: params -using Distributed -using SharedArrays +using BSON +using Flux +using JuMP -""" -bound_tightening(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false) +BSON.@load string(@__DIR__)*"/NN_paraboloid.bson" model -A single-threaded implementation of optimal tightened constraint bounds L and U for for a trained DNN. -Using these bounds with the create_JuMP_model function reduces solution time for optimization problems. +NN_model = model -# Arguments -- `DNN::Chain`: A trained ReLU DNN. -- `init_U_bounds::Vector{Float32}`: Initial upper bounds on the node values of the DNN. -- `init_L_bounds::Vector{Float32}`: Initial lower bounds on the node values of the DNN. -- `verbose::Bool=false`: Controls Gurobi logs. -- `tl::Float64=1.0`: Controls the time limit for solvign the subproblems +init_ub = [1.0f0, 1.0f0] +init_lb = [-1.0f0, -1.0f0] -# Examples -```julia -L_bounds, U_bounds = bound_tightening(DNN, init_U_bounds, init_L_bounds, false, 1.0) -``` -""" -function bound_tightening(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false, tl::Float64=1.0) +# Function begins - K = length(DNN) # NOTE! there are K+1 layers in the nn +K = length(NN_model) # number of layers (input layer not included) +W = [Flux.params(NN_model)[2*k-1] for k in 1:K] +b = [Flux.params(NN_model)[2*k] for k in 1:K] - # store the DNN weights and biases - DNN_params = params(DNN) - W = [DNN_params[2*i-1] for i in 1:K] - b = [DNN_params[2*i] for i in 1:K] +input_length = Int((length(W[1]) / length(b[1]))) +neuron_count = [length(b[k]) for k in eachindex(b)] - # stores the node count of layer k (starting at layer k=0) at index k+1 - input_node_count = length(DNN_params[1][1, :]) - node_count = [if k == 1 input_node_count else length(DNN_params[2*(k-1)]) end for k in 1:K+1] +neurons(layer) = layer == 0 ? [i for i in 1:input_length] : [i for i in 1:neuron_count[layer]] - # store the current optimal bounds in the algorithm - curr_U_bounds = copy(init_U_bounds) - curr_L_bounds = copy(init_L_bounds) +@assert input_length == length(init_ub) == length(init_lb) "Initial bounds arrays must be the same length as the input layer" - model = Model(optimizer_with_attributes(Gurobi.Optimizer, "OutputFlag" => (verbose ? 1 : 0), "TimeLimit" => tl)) +# build model up to second layer +jump_model = JuMP.Model() - # keeps track of the current node index starting from layer 1 (out of 0:K) - outer_index = node_count[1] + 1 +@variable(jump_model, x[layer = 0:K, neurons(layer)]) +@variable(jump_model, s[layer = 0:K, neurons(layer)]) +@variable(jump_model, z[layer = 0:K, neurons(layer)]) - # NOTE! below variables and constraints for all opt problems - @variable(model, x[k in 0:K, j in 1:node_count[k+1]] >= 0) - @variable(model, s[k in 1:K-1, j in 1:node_count[k+1]] >= 0) - @variable(model, z[k in 1:K-1, j in 1:node_count[k+1]], Bin) - @variable(model, U[k in 0:K, j in 1:node_count[k+1]]) - @variable(model, L[k in 0:K, j in 1:node_count[k+1]]) +@constraint(jump_model, [j = 1:input_length], init_lb[j] <= x[0, j] <= init_ub[j]) - # fix values to all U[k,j] and L[k,j] from U_bounds and L_bounds - index = 1 - for k in 0:K - for j in 1:node_count[k+1] - fix(U[k, j], curr_U_bounds[index], force=true) - fix(L[k, j], curr_L_bounds[index], force=true) - index += 1 - end - end - - # input layer (layer 0) node bounds are given beforehand - for input_node in 1:node_count[1] - delete_lower_bound(x[0, input_node]) - @constraint(model, L[0, input_node] <= x[0, input_node]) - @constraint(model, x[0, input_node] <= U[0, input_node]) - end - - # deleting lower bound for output nodes - for output_node in 1:node_count[K+1] - delete_lower_bound(x[K, output_node]) - end - - # NOTE! below constraints depending on the layer - for k in 1:K - # we only want to build ALL of the constraints until the PREVIOUS layer, and then go node by node - # here we calculate ONLY the constraints until the PREVIOUS layer - for node_in in 1:node_count[k] - if k >= 2 - temp_sum = sum(W[k-1][node_in, j] * x[k-1-1, j] for j in 1:node_count[k-1]) - @constraint(model, x[k-1, node_in] <= U[k-1, node_in] * z[k-1, node_in]) - @constraint(model, s[k-1, node_in] <= -L[k-1, node_in] * (1 - z[k-1, node_in])) - if k <= K - 1 - @constraint(model, temp_sum + b[k-1][node_in] == x[k-1, node_in] - s[k-1, node_in]) - else # k == K - @constraint(model, temp_sum + b[k-1][node_in] == x[k-1, node_in]) - end - end - end - - # NOTE! below constraints depending on the node - for node in 1:node_count[k+1] - # here we calculate the specific constraints depending on the current node - temp_sum = sum(W[k][node, j] * x[k-1, j] for j in 1:node_count[k]) # NOTE! prev layer [k] - if k <= K - 1 - @constraint(model, node_con, temp_sum + b[k][node] == x[k, node] - s[k, node]) - @constraint(model, node_U, x[k, node] <= U[k, node] * z[k, node]) - @constraint(model, node_L, s[k, node] <= -L[k, node] * (1 - z[k, node])) - elseif k == K # == last value of k - @constraint(model, node_con, temp_sum + b[k][node] == x[k, node]) - @constraint(model, node_L, L[k, node] <= x[k, node]) - @constraint(model, node_U, x[k, node] <= U[k, node]) - end - - # NOTE! below objective function and optimizing the model depending on obj_function and layer - for obj_function in 1:2 - if obj_function == 1 && k <= K - 1 # Min, hidden layer - @objective(model, Min, x[k, node] - s[k, node]) - elseif obj_function == 2 && k <= K - 1 # Max, hidden layer - @objective(model, Max, x[k, node] - s[k, node]) - elseif obj_function == 1 && k == K # Min, last layer - @objective(model, Min, x[k, node]) - elseif obj_function == 2 && k == K # Max, last layer - @objective(model, Max, x[k, node]) - end - - solve_time = @elapsed optimize!(model) - solve_time = round(solve_time; sigdigits = 3) - # @assert termination_status(model) == OPTIMAL || termination_status(model) == TIME_LIMIT - # "Problem (layer $k (from 1:$K), node $node, $(obj_function == 1 ? "L" : "U")-bound) is infeasible." - optimal = objective_value(model) - println("Layer $k, node $node, $(obj_function == 1 ? "L" : "U")-bound: solve time $(solve_time)s, optimal value $(optimal)") - - # fix the model variable L or U corresponding to the current node to be the optimal value - if obj_function == 1 # Min - curr_L_bounds[outer_index] = optimal - fix(L[k, node], optimal) - elseif obj_function == 2 # Max - curr_U_bounds[outer_index] = optimal - fix(U[k, node], optimal) - end - end - outer_index += 1 - - # deleting and unregistering the constraints assigned to the current node - delete(model, node_con) - delete(model, node_L) - delete(model, node_U) - unregister(model, :node_con) - unregister(model, :node_L) - unregister(model, :node_U) - end - end - - println("Solving optimal constraint bounds single-threaded complete") - - return curr_U_bounds, curr_L_bounds -end - -""" -bound_tightening_threads(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false, tl::float64=1) - -A multi-threaded (using Threads) implementation of optimal tightened constraint bounds L and U for for a trained DNN. -Using these bounds with the create_JuMP_model function reduces solution time for optimization problems. - -# Arguments -- `DNN::Chain`: A trained ReLU DNN. -- `init_U_bounds::Vector{Float32}`: Initial upper bounds on the node values of the DNN. -- `init_L_bounds::Vector{Float32}`: Initial lower bounds on the node values of the DNN. -- `verbose::Bool=false`: Controls Gurobi logs. -- `tl::Float64=1.0`: Controls the time limit for solvign the subproblems - -# Examples -```julia -L_bounds_threads, U_bounds_threads = bound_tightening_threads(DNN, init_U_bounds, init_L_bounds, false, 1.0) -``` -""" -function bound_tightening_threads(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false, tl::Float64=1.0) - - K = length(DNN) # NOTE! there are K+1 layers in the nn - - # store the DNN weights and biases - DNN_params = params(DNN) - W = [DNN_params[2*i-1] for i in 1:K] - b = [DNN_params[2*i] for i in 1:K] - - # stores the node count of layer k (starting at layer k=0) at index k+1 - input_node_count = length(DNN_params[1][1, :]) - node_count = [if k == 1 input_node_count else length(DNN_params[2*(k-1)]) end for k in 1:K+1] - - # store the current optimal bounds in the algorithm - curr_U_bounds = copy(init_U_bounds) - curr_L_bounds = copy(init_L_bounds) - - lock = Threads.ReentrantLock() - - for k in 1:K - - Threads.@threads for node in 1:(2*node_count[k+1]) # loop over both obj functions - - ### below variables and constraints in all problems - - model = Model(optimizer_with_attributes(Gurobi.Optimizer, "OutputFlag" => (verbose ? 1 : 0), "TimeLimit" => tl)) - - # keeps track of the current node index starting from layer 1 (out of 0:K) - prev_layers_node_sum = 0 - for prev_layer in 0:k-1 - prev_layers_node_sum += node_count[prev_layer+1] - end - - # loops nodes twice: 1st time with obj function Min, 2nd time with Max - curr_node = node - obj_function = 1 - if node > node_count[k+1] - curr_node = node - node_count[k+1] - obj_function = 2 - end - curr_node_index = prev_layers_node_sum + curr_node +for layer in 1:K # hidden layers and output layer - second layer and up - # NOTE! below variables and constraints for all opt problems - @variable(model, x[k in 0:K, j in 1:node_count[k+1]] >= 0) - @variable(model, s[k in 1:K-1, j in 1:node_count[k+1]] >= 0) - @variable(model, z[k in 1:K-1, j in 1:node_count[k+1]], Bin) - @variable(model, U[k in 0:K, j in 1:node_count[k+1]]) - @variable(model, L[k in 0:K, j in 1:node_count[k+1]]) + ub_x = [] + ub_s = [] - # fix values to all U[k,j] and L[k,j] from U_bounds and L_bounds - index = 1 - Threads.lock(lock) do - for k in 0:K - for j in 1:node_count[k+1] - fix(U[k, j], curr_U_bounds[index], force=true) - fix(L[k, j], curr_L_bounds[index], force=true) - index += 1 - end - end - end + # TODO: the model must be copied for each neuron in a new layer - # input layer (layer 0) node bounds are given beforehand - for input_node in 1:node_count[1] - delete_lower_bound(x[0, input_node]) - @constraint(model, L[0, input_node] <= x[0, input_node]) - @constraint(model, x[0, input_node] <= U[0, input_node]) - end + for neuron in 1:neuron_count[layer] - # deleting lower bound for output nodes - for output_node in 1:node_count[K+1] - delete_lower_bound(x[K, output_node]) - end + @constraint(jump_model, x[layer, neuron] >= 0) + @constraint(jump_model, s[layer, neuron] >= 0) + set_binary(z[layer, neuron]) - ### below constraints depending on the layer (every constraint up to the previous layer) - for k_in in 1:k - for node_in in 1:node_count[k_in] - if k_in >= 2 - temp_sum = sum(W[k_in-1][node_in, j] * x[k_in-1-1, j] for j in 1:node_count[k_in-1]) - @constraint(model, x[k_in-1, node_in] <= U[k_in-1, node_in] * z[k_in-1, node_in]) - @constraint(model, s[k_in-1, node_in] <= -L[k_in-1, node_in] * (1 - z[k_in-1, node_in])) - if k_in <= K - 1 - @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in] - s[k_in-1, node_in]) - else # k_in == K - @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in]) - end - end - end - end + @constraint(jump_model, z[layer, neuron] --> {x[layer, neuron] <= 0}) + @constraint(jump_model, !z[layer, neuron] --> {s[layer, neuron] <= 0}) - ### below constraints depending on the node - temp_sum = sum(W[k][curr_node, j] * x[k-1, j] for j in 1:node_count[k]) # NOTE! prev layer [k] - if k <= K - 1 - @constraint(model, node_con, temp_sum + b[k][curr_node] == x[k, curr_node] - s[k, curr_node]) - @constraint(model, node_U, x[k, curr_node] <= U[k, curr_node] * z[k, curr_node]) - @constraint(model, node_L, s[k, curr_node] <= -L[k, curr_node] * (1 - z[k, curr_node])) - elseif k == K # == last value of k - @constraint(model, node_con, temp_sum + b[k][curr_node] == x[k, curr_node]) - @constraint(model, node_L, L[k, curr_node] <= x[k, curr_node]) - @constraint(model, node_U, x[k, curr_node] <= U[k, curr_node]) - end - - if obj_function == 1 && k <= K - 1 # Min, hidden layer - @objective(model, Min, x[k, curr_node] - s[k, curr_node]) - elseif obj_function == 2 && k <= K - 1 # Max, hidden layer - @objective(model, Max, x[k, curr_node] - s[k, curr_node]) - elseif obj_function == 1 && k == K # Min, last layer - @objective(model, Min, x[k, curr_node]) - elseif obj_function == 2 && k == K # Max, last layer - @objective(model, Max, x[k, curr_node]) - end - - solve_time = @elapsed optimize!(model) - solve_time = round(solve_time; sigdigits = 3) - # @assert termination_status(model) == OPTIMAL || termination_status(model) == TIME_LIMIT - # "Problem (layer $k (from 1:$K), node $curr_node, $(obj_function == 1 ? "L" : "U")-bound) is infeasible." - optimal = objective_value(model) - - # @show termination_status(model) - # if termination_status(model) == OPTIMAL - # optimal = objective_value(model) - # else - # optimal = Inf - # end - - - println("Thread: $(Threads.threadid()), layer $k, node $curr_node, $(obj_function == 1 ? "L" : "U")-bound: solve time $(solve_time)s, optimal value $(optimal)") - - # fix the model variable L or U corresponding to the current node to be the optimal value - Threads.lock(lock) do - if obj_function == 1 && optimal != Inf # Min and we recieved a new bound - - curr_L_bounds[curr_node_index] = optimal - fix(L[k, curr_node], optimal) - - elseif obj_function == 2 && optimal != Inf # Max and we recieved a new bound - - curr_U_bounds[curr_node_index] = optimal - fix(U[k, curr_node], optimal) - - end - end - - end - - end - - println("Solving optimal constraint bounds using threads complete") - - return curr_U_bounds, curr_L_bounds -end - -""" -bound_tightening_workers(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false, tl::Float64=1.0) - -A multi-threaded (using workers) implementation of optimal tightened constraint bounds L and U for for a trained DNN. -Using these bounds with the create_JuMP_model function reduces solution time for optimization problems. - -# Arguments -- `DNN::Chain`: A trained ReLU DNN. -- `init_U_bounds::Vector{Float32}`: Initial upper bounds on the node values of the DNN. -- `init_L_bounds::Vector{Float32}`: Initial lower bounds on the node values of the DNN. -- `verbose::Bool=false`: Controls Gurobi logs. -- `tl::Float64=1.0`: Controls the time limit for solvign the subproblems - -# Examples -```julia -L_bounds_workers, U_bounds_workers = bound_tightening_workers(DNN, init_U_bounds, init_L_bounds, false, 1.0) -``` -""" -function bound_tightening_workers(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false, tl::Float64=1.0) - - K = length(DNN) # NOTE! there are K+1 layers in the nn - - # store the DNN weights and biases - DNN_params = params(DNN) - W = [DNN_params[2*i-1] for i in 1:K] - b = [DNN_params[2*i] for i in 1:K] - - # stores the node count of layer k (starting at layer k=0) at index k+1 - input_node_count = length(DNN_params[1][1, :]) - node_count = [if k == 1 input_node_count else length(DNN_params[2*(k-1)]) end for k in 1:K+1] - - # store the current optimal bounds in the algorithm - curr_U_bounds = copy(init_U_bounds) - curr_L_bounds = copy(init_L_bounds) - - for k in 1:K - - # Distributed.pmap returns the bounds in order - L_U_bounds = Distributed.pmap(node -> bt_workers_inner(K, k, node, W, b, node_count, curr_U_bounds, curr_L_bounds, verbose, tl), 1:(2*node_count[k+1])) - - for node in 1:node_count[k+1] - prev_layers_node_sum = 0 - for prev_layer in 0:k-1 - prev_layers_node_sum += node_count[prev_layer+1] - end - - # loops nodes twice: 1st time with obj function Min, 2nd time with Max - curr_node = node - obj_function = 1 - if node > node_count[k+1] - curr_node = node - node_count[k+1] - obj_function = 2 - end - curr_node_index = prev_layers_node_sum + curr_node - - # L-bounds in 1:node_count[k+1], U-bounds in 1:(node + node_count[k+1]) - curr_L_bounds[curr_node_index] = L_U_bounds[node] - curr_U_bounds[curr_node_index] = L_U_bounds[node + node_count[k+1]] - end + @constraint(jump_model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1, i] for i in neurons(layer-1))) + @objective(jump_model, Max, x[layer, neuron]) # ub_x + @objective(jump_model, Max, s[layer, neuron]) # ub_s end - println("Solving optimal constraint bounds using workers complete") - - return curr_U_bounds, curr_L_bounds + # add lower and upper bound constraints for x and s end -# Inner function to bound_tightening_workers: assigns a JuMP model to the current worker - -function bt_workers_inner( - K::Int64, - k::Int64, - node::Int64, - W::Vector{Matrix{Float32}}, - b::Vector{Vector{Float32}}, - node_count::Vector{Int64}, - curr_U_bounds::Vector{Float32}, - curr_L_bounds::Vector{Float32}, - verbose::Bool, - tl::Float64 - ) - - model = Model(optimizer_with_attributes(Gurobi.Optimizer, "OutputFlag" => (verbose ? 1 : 0), "Threads" => 1, "TimeLimit" => tl)) - - # keeps track of the current node index starting from layer 1 (out of 0:K) - prev_layers_node_sum = 0 - for prev_layer in 0:k-1 - prev_layers_node_sum += node_count[prev_layer+1] - end - - # loops nodes twice: 1st time with obj function Min, 2nd time with Max - curr_node = node - obj_function = 1 - if node > node_count[k+1] - curr_node = node - node_count[k+1] - obj_function = 2 - end +return jump_model - # NOTE! below variables and constraints for all opt problems - @variable(model, x[k in 0:K, j in 1:node_count[k+1]] >= 0) - @variable(model, s[k in 1:K-1, j in 1:node_count[k+1]] >= 0) - @variable(model, z[k in 1:K-1, j in 1:node_count[k+1]], Bin) - @variable(model, U[k in 0:K, j in 1:node_count[k+1]]) - @variable(model, L[k in 0:K, j in 1:node_count[k+1]]) - - # fix values to all U[k,j] and L[k,j] from U_bounds and L_bounds - index = 1 - for k in 0:K - for j in 1:node_count[k+1] - fix(U[k, j], curr_U_bounds[index], force=true) - fix(L[k, j], curr_L_bounds[index], force=true) - index += 1 - end - end - - # input layer (layer 0) node bounds are given beforehand - for input_node in 1:node_count[1] - delete_lower_bound(x[0, input_node]) - @constraint(model, L[0, input_node] <= x[0, input_node]) - @constraint(model, x[0, input_node] <= U[0, input_node]) - end - - # deleting lower bound for output nodes - for output_node in 1:node_count[K+1] - delete_lower_bound(x[K, output_node]) - end - - ### below constraints depending on the layer (every constraint up to the previous layer) - for k_in in 1:k - for node_in in 1:node_count[k_in] - if k_in >= 2 - temp_sum = sum(W[k_in-1][node_in, j] * x[k_in-1-1, j] for j in 1:node_count[k_in-1]) - @constraint(model, x[k_in-1, node_in] <= U[k_in-1, node_in] * z[k_in-1, node_in]) - @constraint(model, s[k_in-1, node_in] <= -L[k_in-1, node_in] * (1 - z[k_in-1, node_in])) - if k_in <= K - 1 - @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in] - s[k_in-1, node_in]) - else # k_in == K - @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in]) - end - end - end - end - - ### below constraints depending on the node - temp_sum = sum(W[k][curr_node, j] * x[k-1, j] for j in 1:node_count[k]) # NOTE! prev layer [k] - if k <= K - 1 - @constraint(model, node_con, temp_sum + b[k][curr_node] == x[k, curr_node] - s[k, curr_node]) - @constraint(model, node_U, x[k, curr_node] <= U[k, curr_node] * z[k, curr_node]) - @constraint(model, node_L, s[k, curr_node] <= -L[k, curr_node] * (1 - z[k, curr_node])) - elseif k == K # == last value of k - @constraint(model, node_con, temp_sum + b[k][curr_node] == x[k, curr_node]) - @constraint(model, node_L, L[k, curr_node] <= x[k, curr_node]) - @constraint(model, node_U, x[k, curr_node] <= U[k, curr_node]) - end - - if obj_function == 1 && k <= K - 1 # Min, hidden layer - @objective(model, Min, x[k, curr_node] - s[k, curr_node]) - elseif obj_function == 2 && k <= K - 1 # Max, hidden layer - @objective(model, Max, x[k, curr_node] - s[k, curr_node]) - elseif obj_function == 1 && k == K # Min, last layer - @objective(model, Min, x[k, curr_node]) - elseif obj_function == 2 && k == K # Max, last layer - @objective(model, Max, x[k, curr_node]) - end - - solve_time = @elapsed optimize!(model) - solve_time = round(solve_time; sigdigits = 3) - # @assert termination_status(model) == OPTIMAL || termination_status(model) == TIME_LIMIT - # "Problem (layer $k (from 1:$K), node $curr_node, $(obj_function == 1 ? "L" : "U")-bound) is infeasible." - optimal = objective_value(model) - println("Worker: $(myid()), layer $k, node $curr_node, $(obj_function == 1 ? "L" : "U")-bound: solve time $(solve_time)s, optimal value $(optimal)") - - return optimal -end - -""" -bound_tightening_2workers(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false) - -A multi-threaded (using workers) implementation of optimal tightened constraint bounds L and U for for a trained DNN. -This function uses two in-place models at each layer to reduce memory usage. A max of 2 workers in use simultaneously. -Using these bounds with the create_JuMP_model function reduces solution time for optimization problems. - -# Arguments -- `DNN::Chain`: A trained ReLU DNN. -- `init_U_bounds::Vector{Float32}`: Initial upper bounds on the node values of the DNN. -- `init_L_bounds::Vector{Float32}`: Initial lower bounds on the node values of the DNN. -- `verbose::Bool=false`: Controls Gurobi logs. -- `tl::Float64=1.0`: Controls the time limit for solvign the subproblems. - -# Examples -```julia -L_bounds_workers, U_bounds_workers = bound_tightening_2workers(DNN, init_U_bounds, init_L_bounds, false, 1.0) -``` -""" -function bound_tightening_2workers(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false, tl::Float64=1.0) - - K = length(DNN) # NOTE! there are K+1 layers in the nn - - # store the DNN weights and biases - DNN_params = params(DNN) - W = [DNN_params[2*i-1] for i in 1:K] - b = [DNN_params[2*i] for i in 1:K] - - # stores the node count of layer k (starting at layer k=0) at index k+1 - input_node_count = length(DNN_params[1][1, :]) - node_count = [if k == 1 input_node_count else length(DNN_params[2*(k-1)]) end for k in 1:K+1] - - # store the current optimal bounds in the algorithm - curr_U_bounds = copy(init_U_bounds) - curr_L_bounds = copy(init_L_bounds) - - # split the available threads into 2 to be assigned to each worker (integer division) - n = Threads.nthreads() - threads_split = [n÷2, n-(n÷2)] - - for k in 1:K - - L_U_bounds = Distributed.pmap(obj_function -> - bt_2workers_inner(K, k, obj_function, W, b, node_count, curr_U_bounds, curr_L_bounds, threads_split[obj_function], verbose, tl), 1:2) - - curr_L_bounds = L_U_bounds[1] - curr_U_bounds = L_U_bounds[2] - - end - - println("Solving optimal constraint bounds complete") - - return curr_U_bounds, curr_L_bounds -end - - -# Inner function to solve_optimal_bounds_2workers: solves L or U bounds for all nodes in a layer using the same JuMP model - -function bt_2workers_inner( - K::Int64, - k::Int64, - obj_function::Int64, - W::Vector{Matrix{Float32}}, - b::Vector{Vector{Float32}}, - node_count::Vector{Int64}, - curr_U_bounds::Vector{Float32}, - curr_L_bounds::Vector{Float32}, - n_threads::Int64, - verbose::Bool, - tl::Float64 - ) - - curr_U_bounds_copy = copy(curr_U_bounds) - curr_L_bounds_copy = copy(curr_L_bounds) - - model = Model(optimizer_with_attributes(Gurobi.Optimizer, "OutputFlag" => (verbose ? 1 : 0), "Threads" => n_threads, "TimeLimit" => tl)) - - # NOTE! below variables and constraints for all opt problems - @variable(model, x[k in 0:K, j in 1:node_count[k+1]] >= 0) - @variable(model, s[k in 1:K-1, j in 1:node_count[k+1]] >= 0) - @variable(model, z[k in 1:K-1, j in 1:node_count[k+1]], Bin) - @variable(model, U[k in 0:K, j in 1:node_count[k+1]]) - @variable(model, L[k in 0:K, j in 1:node_count[k+1]]) - - # fix values to all U[k,j] and L[k,j] from U_bounds and L_bounds - index = 1 - for k in 0:K - for j in 1:node_count[k+1] - fix(U[k, j], curr_U_bounds[index], force=true) - fix(L[k, j], curr_L_bounds[index], force=true) - index += 1 - end - end - - # input layer (layer 0) node bounds are given beforehand - for input_node in 1:node_count[1] - delete_lower_bound(x[0, input_node]) - @constraint(model, L[0, input_node] <= x[0, input_node]) - @constraint(model, x[0, input_node] <= U[0, input_node]) - end - - # deleting lower bound for output nodes - for output_node in 1:node_count[K+1] - delete_lower_bound(x[K, output_node]) - end - - ### below constraints depending on the layer (every constraint up to the previous layer) - for k_in in 1:k - for node_in in 1:node_count[k_in] - if k_in >= 2 - temp_sum = sum(W[k_in-1][node_in, j] * x[k_in-1-1, j] for j in 1:node_count[k_in-1]) - @constraint(model, x[k_in-1, node_in] <= U[k_in-1, node_in] * z[k_in-1, node_in]) - @constraint(model, s[k_in-1, node_in] <= -L[k_in-1, node_in] * (1 - z[k_in-1, node_in])) - if k_in <= K - 1 - @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in] - s[k_in-1, node_in]) - else # k_in == K - @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in]) - end - end - end - end - - for node in 1:node_count[k+1] - - prev_layers_node_sum = 0 - for prev_layer in 0:k-1 - prev_layers_node_sum += node_count[prev_layer+1] - end - curr_node_index = prev_layers_node_sum + node - - ### below constraints depending on the node - temp_sum = sum(W[k][node, j] * x[k-1, j] for j in 1:node_count[k]) # NOTE! prev layer [k] - if k <= K - 1 - @constraint(model, node_con, temp_sum + b[k][node] == x[k, node] - s[k, node]) - @constraint(model, node_U, x[k, node] <= U[k, node] * z[k, node]) - @constraint(model, node_L, s[k, node] <= -L[k, node] * (1 - z[k, node])) - elseif k == K # == last value of k - @constraint(model, node_con, temp_sum + b[k][node] == x[k, node]) - @constraint(model, node_L, L[k, node] <= x[k, node]) - @constraint(model, node_U, x[k, node] <= U[k, node]) - end - - if obj_function == 1 && k <= K - 1 # Min, hidden layer - @objective(model, Min, x[k, node] - s[k, node]) - elseif obj_function == 2 && k <= K - 1 # Max, hidden layer - @objective(model, Max, x[k, node] - s[k, node]) - elseif obj_function == 1 && k == K # Min, last layer - @objective(model, Min, x[k, node]) - elseif obj_function == 2 && k == K # Max, last layer - @objective(model, Max, x[k, node]) - end - - solve_time = @elapsed optimize!(model) - solve_time = round(solve_time; sigdigits = 3) - @assert termination_status(model) == OPTIMAL || termination_status(model) == TIME_LIMIT - "Problem (layer $k (from 1:$K), node $node, $(obj_function == 1 ? "L" : "U")-bound) is infeasible." - optimal = objective_value(model) - println("Worker: $(myid()), layer $k, node $node, $(obj_function == 1 ? "L" : "U")-bound: solve time $(solve_time)s, optimal value $(optimal)") - - # fix the model variable L or U corresponding to the current node to be the optimal value - if obj_function == 1 # Min - curr_L_bounds_copy[curr_node_index] = optimal - elseif obj_function == 2 # Max - curr_U_bounds_copy[curr_node_index] = optimal - end - - # deleting and unregistering the constraints assigned to the current node - delete(model, node_con) - delete(model, node_L) - delete(model, node_U) - unregister(model, :node_con) - unregister(model, :node_L) - unregister(model, :node_U) - end - - if obj_function == 1 # Min - return curr_L_bounds_copy - elseif obj_function == 2 # Max - return curr_U_bounds_copy - end +function bound_tightening(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}) end \ No newline at end of file diff --git a/test/tree_ensembles/TE_training.jl b/test/tree_ensembles/paraboloid_training.jl similarity index 100% rename from test/tree_ensembles/TE_training.jl rename to test/tree_ensembles/paraboloid_training.jl From 75f7518c5fdb7093b054626c288bd527e28e28a2 Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Tue, 23 Jan 2024 19:05:37 +0200 Subject: [PATCH 02/32] completely rebuilt bound tightening from the ground up --- src/neural_networks/bound_tightening_new.jl | 116 +++++++++++--------- 1 file changed, 65 insertions(+), 51 deletions(-) diff --git a/src/neural_networks/bound_tightening_new.jl b/src/neural_networks/bound_tightening_new.jl index b0dd1a0..1d0dc25 100644 --- a/src/neural_networks/bound_tightening_new.jl +++ b/src/neural_networks/bound_tightening_new.jl @@ -1,63 +1,77 @@ using BSON using Flux using JuMP +using Gurobi BSON.@load string(@__DIR__)*"/NN_paraboloid.bson" model -NN_model = model - init_ub = [1.0f0, 1.0f0] init_lb = [-1.0f0, -1.0f0] -# Function begins - -K = length(NN_model) # number of layers (input layer not included) -W = [Flux.params(NN_model)[2*k-1] for k in 1:K] -b = [Flux.params(NN_model)[2*k] for k in 1:K] - -input_length = Int((length(W[1]) / length(b[1]))) -neuron_count = [length(b[k]) for k in eachindex(b)] - -neurons(layer) = layer == 0 ? [i for i in 1:input_length] : [i for i in 1:neuron_count[layer]] - -@assert input_length == length(init_ub) == length(init_lb) "Initial bounds arrays must be the same length as the input layer" - -# build model up to second layer -jump_model = JuMP.Model() - -@variable(jump_model, x[layer = 0:K, neurons(layer)]) -@variable(jump_model, s[layer = 0:K, neurons(layer)]) -@variable(jump_model, z[layer = 0:K, neurons(layer)]) - -@constraint(jump_model, [j = 1:input_length], init_lb[j] <= x[0, j] <= init_ub[j]) - -for layer in 1:K # hidden layers and output layer - second layer and up - - ub_x = [] - ub_s = [] - - # TODO: the model must be copied for each neuron in a new layer - - for neuron in 1:neuron_count[layer] - - @constraint(jump_model, x[layer, neuron] >= 0) - @constraint(jump_model, s[layer, neuron] >= 0) - set_binary(z[layer, neuron]) - - @constraint(jump_model, z[layer, neuron] --> {x[layer, neuron] <= 0}) - @constraint(jump_model, !z[layer, neuron] --> {s[layer, neuron] <= 0}) - - @constraint(jump_model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1, i] for i in neurons(layer-1))) - - @objective(jump_model, Max, x[layer, neuron]) # ub_x - @objective(jump_model, Max, s[layer, neuron]) # ub_s +jump_model, bounds_x, bounds_s = bound_tightening(model, init_ub, init_lb) +bounds_x +bounds_s +objective_function(jump_model) + +function bound_tightening(NN_model::Flux.Chain, init_ub::Vector{Float32}, init_lb::Vector{Float32}) + + K = length(NN_model) # number of layers (input layer not included) + W = [Flux.params(NN_model)[2*k-1] for k in 1:K] + b = [Flux.params(NN_model)[2*k] for k in 1:K] + + input_length = Int((length(W[1]) / length(b[1]))) + neuron_count = [length(b[k]) for k in eachindex(b)] + neurons(layer) = layer == 0 ? [i for i in 1:input_length] : [i for i in 1:neuron_count[layer]] + + @assert input_length == length(init_ub) == length(init_lb) "Initial bounds arrays must be the same length as the input layer" + + # build model up to second layer + jump_model = direct_model(Gurobi.Optimizer()) + set_silent(jump_model) + + @variable(jump_model, x[layer = 0:K, neurons(layer)]) + @variable(jump_model, s[layer = 0:K, neurons(layer)]) + @variable(jump_model, z[layer = 0:K, neurons(layer)]) + + @constraint(jump_model, [j = 1:input_length], x[0, j] <= init_ub[j]) + @constraint(jump_model, [j = 1:input_length], x[0, j] >= init_lb[j]) + + bounds_x = Vector{Vector}(undef, K) + bounds_s = Vector{Vector}(undef, K) + + for layer in 1:K # hidden layers to output layer - second layer and up + + ub_x = Vector{Float32}(undef, length(neurons(layer))) + ub_s = Vector{Float32}(undef, length(neurons(layer))) + + # TODO: For parallelization the model must be copied for each neuron in a new layer to prevent data races + + for neuron in 1:neuron_count[layer] + + @constraint(jump_model, x[layer, neuron] >= 0) + @constraint(jump_model, s[layer, neuron] >= 0) + set_binary(z[layer, neuron]) + + @constraint(jump_model, z[layer, neuron] --> {x[layer, neuron] <= 0}) + @constraint(jump_model, !z[layer, neuron] --> {s[layer, neuron] <= 0}) + + @constraint(jump_model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1, i] for i in neurons(layer-1))) + + @objective(jump_model, Max, x[layer, neuron]) + optimize!(jump_model) + ub_x[neuron] = objective_value(jump_model) + + @objective(jump_model, Max, s[layer, neuron]) + optimize!(jump_model) + ub_s[neuron] = objective_value(jump_model) + end + + bounds_x[layer] = ub_x + bounds_s[layer] = ub_s + + @constraint(jump_model, [neuron = neurons(layer)], x[layer, neuron] <= ub_x[neuron]) + @constraint(jump_model, [neuron = neurons(layer)], s[layer, neuron] <= ub_s[neuron]) end - # add lower and upper bound constraints for x and s -end - -return jump_model - -function bound_tightening(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}) - + return jump_model, bounds_x, bounds_s end \ No newline at end of file From 01b500bb3631c4cc70c61c41fd87504629e09d30 Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Wed, 24 Jan 2024 10:54:13 +0200 Subject: [PATCH 03/32] fully working bound tightening --- src/neural_networks/bound_tightening_new.jl | 41 +++++++++++++++++---- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/src/neural_networks/bound_tightening_new.jl b/src/neural_networks/bound_tightening_new.jl index 1d0dc25..8d0fbfa 100644 --- a/src/neural_networks/bound_tightening_new.jl +++ b/src/neural_networks/bound_tightening_new.jl @@ -5,15 +5,25 @@ using Gurobi BSON.@load string(@__DIR__)*"/NN_paraboloid.bson" model -init_ub = [1.0f0, 1.0f0] -init_lb = [-1.0f0, -1.0f0] +init_ub = [1.0, 1.0] +init_lb = [-1.0, -1.0] -jump_model, bounds_x, bounds_s = bound_tightening(model, init_ub, init_lb) -bounds_x -bounds_s -objective_function(jump_model) +model = Chain( + Dense(2 => 30, relu), + Dense(30 => 50, relu), + Dense(50 => 1, relu) +) -function bound_tightening(NN_model::Flux.Chain, init_ub::Vector{Float32}, init_lb::Vector{Float32}) +@time jump_model, bounds_x, bounds_s = bound_tightening(model, init_ub, init_lb) + +data = rand(Float32, (2, 1000)) .- 0.5f0 + +x_train = data[:, 1:750] +y_train = [sum(x_train[:, col].^2) for col in 1:750] + +vec(model(x_train)) ≈ [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750] + +function bound_tightening(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}) K = length(NN_model) # number of layers (input layer not included) W = [Flux.params(NN_model)[2*k-1] for k in 1:K] @@ -40,6 +50,8 @@ function bound_tightening(NN_model::Flux.Chain, init_ub::Vector{Float32}, init_l bounds_s = Vector{Vector}(undef, K) for layer in 1:K # hidden layers to output layer - second layer and up + + println("LAYER $layer") ub_x = Vector{Float32}(undef, length(neurons(layer))) ub_s = Vector{Float32}(undef, length(neurons(layer))) @@ -47,6 +59,8 @@ function bound_tightening(NN_model::Flux.Chain, init_ub::Vector{Float32}, init_l # TODO: For parallelization the model must be copied for each neuron in a new layer to prevent data races for neuron in 1:neuron_count[layer] + + print("#") @constraint(jump_model, x[layer, neuron] >= 0) @constraint(jump_model, s[layer, neuron] >= 0) @@ -65,6 +79,8 @@ function bound_tightening(NN_model::Flux.Chain, init_ub::Vector{Float32}, init_l optimize!(jump_model) ub_s[neuron] = objective_value(jump_model) end + + println() bounds_x[layer] = ub_x bounds_s[layer] = ub_s @@ -74,4 +90,15 @@ function bound_tightening(NN_model::Flux.Chain, init_ub::Vector{Float32}, init_l end return jump_model, bounds_x, bounds_s +end + +function forward_pass!(jump_model::JuMP.Model, input::Vector{Float32}) + @assert length(input) == length(jump_model[:x][0, :]) "Incorrect input length." + + [fix(jump_model[:x][0, i], input[i], force=true) for i in eachindex(input)] + optimize!(jump_model) + + (last_layer, outputs) = maximum(keys(jump_model[:x].data)) + result = value.(jump_model[:x][last_layer, :]) + return [result[i] for i in 1:outputs] end \ No newline at end of file From 9a449eb604611ddd171ef9556cc2bd210750427b Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Wed, 24 Jan 2024 16:05:05 +0200 Subject: [PATCH 04/32] optimized bound tightening code --- Project.toml | 2 + src/neural_networks/NN_test.jl | 36 ++++++ src/neural_networks/bound_tightening_new.jl | 124 +++++++++++++------- 3 files changed, 120 insertions(+), 42 deletions(-) create mode 100644 src/neural_networks/NN_test.jl diff --git a/Project.toml b/Project.toml index d435756..c6165bb 100644 --- a/Project.toml +++ b/Project.toml @@ -5,6 +5,7 @@ version = "1.0.0-DEV" [deps] BSON = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0" +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" EvoTrees = "f6006082-12f8-11e9-0c9c-0d5d367ab1e5" Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" @@ -14,6 +15,7 @@ JuMP = "4076af6c-e467-56ae-b986-b466b2749572" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" MLDatasets = "eb30cadb-4394-5ae3-aed4-317e484a6458" NPZ = "15e1cf62-19b3-5cfa-8e77-841668bca605" +PProf = "e4faabce-9ead-11e9-39d9-4379958e3056" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SCIP = "82193955-e24f-5292-bf16-6f2c5261a85f" diff --git a/src/neural_networks/NN_test.jl b/src/neural_networks/NN_test.jl new file mode 100644 index 0000000..711eb66 --- /dev/null +++ b/src/neural_networks/NN_test.jl @@ -0,0 +1,36 @@ +include("bound_tightening_new.jl") +BSON.@load string(@__DIR__)*"/NN_paraboloid.bson" model + +init_ub = [1.0, 1.0] +init_lb = [-1.0, -1.0] + +model = Chain( + Dense(2 => 30, relu), + Dense(30 => 50, relu), + Dense(50 => 1, relu) +) + +x_correct = [ + Float32[1.3650875, 1.6169983, 1.1242903, 1.3936517], + Float32[0.9487423, -0.0, 0.8809887], + Float32[1.0439509] +] + +s_correct = [ + Float32[1.6752996, 2.0683866, 0.40223768, 1.4051342], + Float32[0.36901128, 0.76332575, 0.47233626], + Float32[-0.0] +] + +const ENV = Gurobi.Env(); + +@time jump_model, bounds_x, bounds_s = NN_to_MIP(model, init_ub, init_lb, ENV; tighten_bounds=true); +jump_model +bounds_x +bounds_s + +data = rand(Float32, (2, 1000)) .- 0.5f0; +x_train = data[:, 1:750]; +y_train = [sum(x_train[:, col].^2) for col in 1:750]; +@time [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750]; +vec(model(x_train)) ≈ [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750] \ No newline at end of file diff --git a/src/neural_networks/bound_tightening_new.jl b/src/neural_networks/bound_tightening_new.jl index 8d0fbfa..0e59848 100644 --- a/src/neural_networks/bound_tightening_new.jl +++ b/src/neural_networks/bound_tightening_new.jl @@ -3,27 +3,7 @@ using Flux using JuMP using Gurobi -BSON.@load string(@__DIR__)*"/NN_paraboloid.bson" model - -init_ub = [1.0, 1.0] -init_lb = [-1.0, -1.0] - -model = Chain( - Dense(2 => 30, relu), - Dense(30 => 50, relu), - Dense(50 => 1, relu) -) - -@time jump_model, bounds_x, bounds_s = bound_tightening(model, init_ub, init_lb) - -data = rand(Float32, (2, 1000)) .- 0.5f0 - -x_train = data[:, 1:750] -y_train = [sum(x_train[:, col].^2) for col in 1:750] - -vec(model(x_train)) ≈ [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750] - -function bound_tightening(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}) +function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}, environment; tighten_bounds=true) K = length(NN_model) # number of layers (input layer not included) W = [Flux.params(NN_model)[2*k-1] for k in 1:K] @@ -36,7 +16,8 @@ function bound_tightening(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_l @assert input_length == length(init_ub) == length(init_lb) "Initial bounds arrays must be the same length as the input layer" # build model up to second layer - jump_model = direct_model(Gurobi.Optimizer()) + jump_model = Model() + set_optimizer(jump_model, () -> Gurobi.Optimizer(environment)) set_silent(jump_model) @variable(jump_model, x[layer = 0:K, neurons(layer)]) @@ -50,37 +31,28 @@ function bound_tightening(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_l bounds_s = Vector{Vector}(undef, K) for layer in 1:K # hidden layers to output layer - second layer and up - - println("LAYER $layer") - ub_x = Vector{Float32}(undef, length(neurons(layer))) - ub_s = Vector{Float32}(undef, length(neurons(layer))) + ub_x = fill(1000.0, length(neurons(layer))) + ub_s = fill(1000.0, length(neurons(layer))) # TODO: For parallelization the model must be copied for each neuron in a new layer to prevent data races - + + for neuron in 1:neuron_count[layer] + if tighten_bounds ub_x[neuron], ub_s[neuron] = calculate_bounds(jump_model, layer, neuron, W, b, neurons) end + end + for neuron in 1:neuron_count[layer] - print("#") - @constraint(jump_model, x[layer, neuron] >= 0) @constraint(jump_model, s[layer, neuron] >= 0) set_binary(z[layer, neuron]) - + @constraint(jump_model, z[layer, neuron] --> {x[layer, neuron] <= 0}) @constraint(jump_model, !z[layer, neuron] --> {s[layer, neuron] <= 0}) - + @constraint(jump_model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1, i] for i in neurons(layer-1))) - - @objective(jump_model, Max, x[layer, neuron]) - optimize!(jump_model) - ub_x[neuron] = objective_value(jump_model) - - @objective(jump_model, Max, s[layer, neuron]) - optimize!(jump_model) - ub_s[neuron] = objective_value(jump_model) - end - println() + end bounds_x[layer] = ub_x bounds_s[layer] = ub_s @@ -89,7 +61,75 @@ function bound_tightening(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_l @constraint(jump_model, [neuron = neurons(layer)], s[layer, neuron] <= ub_s[neuron]) end - return jump_model, bounds_x, bounds_s + return jump_model, bounds_x, bounds_s, opt_time +end + +function calculate_bounds(model::JuMP.Model, layer, neuron, W, b, neurons) + + x = model[:x] + s = model[:s] + z = model[:z] + + @constraint(model, x_con, x[layer, neuron] >= 0) + @constraint(model, s_con, s[layer, neuron] >= 0) + set_binary(z[layer, neuron]) + + @constraint(model, zx_con, z[layer, neuron] --> {x[layer, neuron] <= 0}) + @constraint(model, zs_con, !z[layer, neuron] --> {s[layer, neuron] <= 0}) + + @constraint(model, w_con, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1, i] for i in neurons(layer-1))) + + @objective(model, Max, x[layer, neuron]) + optimize!(model) + ub_x = objective_value(model) + + @objective(model, Max, s[layer, neuron]) + optimize!(model) + ub_s = objective_value(model) + + delete(model, x_con) + delete(model, s_con) + delete(model, zx_con) + delete(model, zs_con) + delete(model, w_con) + unregister(model, :x_con) + unregister(model, :s_con) + unregister(model, :zx_con) + unregister(model, :zs_con) + unregister(model, :w_con) + unset_binary(z[layer, neuron]) + + return ub_x, ub_s +end + +function calculate_bounds_copy(input_model::JuMP.Model, layer, neuron, W, b, neurons, environment) + + model = copy(input_model) + set_optimizer(model, () -> Gurobi.Optimizer(environment)) + set_silent(model) + + x = model[:x] + s = model[:s] + z = model[:z] + + @constraint(model, x[layer, neuron] >= 0) + @constraint(model, s[layer, neuron] >= 0) + set_binary(z[layer, neuron]) + + @constraint(model, z[layer, neuron] --> {x[layer, neuron] <= 0}) + @constraint(model, !z[layer, neuron] --> {s[layer, neuron] <= 0}) + + @constraint(model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1, i] for i in neurons(layer-1))) + + @objective(model, Max, x[layer, neuron]) + optimize!(model) + ub_x = objective_value(model) + + @objective(model, Max, s[layer, neuron]) + optimize!(model) + ub_s = objective_value(model) + + return ub_x, ub_s end function forward_pass!(jump_model::JuMP.Model, input::Vector{Float32}) From a12750792046127793e55e9084889e84a68ad3e6 Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Wed, 24 Jan 2024 16:50:44 +0200 Subject: [PATCH 05/32] non-working distribution parallelization --- src/neural_networks/NN_test.jl | 37 ++++++++++----------- src/neural_networks/bound_tightening_new.jl | 13 ++++---- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/neural_networks/NN_test.jl b/src/neural_networks/NN_test.jl index 711eb66..e2f0368 100644 --- a/src/neural_networks/NN_test.jl +++ b/src/neural_networks/NN_test.jl @@ -1,26 +1,22 @@ -include("bound_tightening_new.jl") -BSON.@load string(@__DIR__)*"/NN_paraboloid.bson" model +using Distributed + +addprocs(2, exeflags=["--project"]) + +@everywhere include("bound_tightening_new.jl") +#BSON.@load string(@__DIR__)*"/NN_paraboloid.bson" model init_ub = [1.0, 1.0] init_lb = [-1.0, -1.0] +data = rand(Float32, (2, 1000)) .- 0.5f0; +x_train = data[:, 1:750]; +y_train = [sum(x_train[:, col].^2) for col in 1:750]; + model = Chain( Dense(2 => 30, relu), Dense(30 => 50, relu), Dense(50 => 1, relu) -) - -x_correct = [ - Float32[1.3650875, 1.6169983, 1.1242903, 1.3936517], - Float32[0.9487423, -0.0, 0.8809887], - Float32[1.0439509] -] - -s_correct = [ - Float32[1.6752996, 2.0683866, 0.40223768, 1.4051342], - Float32[0.36901128, 0.76332575, 0.47233626], - Float32[-0.0] -] +) const ENV = Gurobi.Env(); @@ -28,9 +24,12 @@ const ENV = Gurobi.Env(); jump_model bounds_x bounds_s +@time [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750]; +vec(model(x_train)) ≈ [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750] -data = rand(Float32, (2, 1000)) .- 0.5f0; -x_train = data[:, 1:750]; -y_train = [sum(x_train[:, col].^2) for col in 1:750]; +@time jump_model, bounds_x, bounds_s = NN_to_MIP(model, init_ub, init_lb, ENV; tighten_bounds=false); +jump_model +bounds_x +bounds_s @time [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750]; -vec(model(x_train)) ≈ [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750] \ No newline at end of file +vec(model(x_train)) ≈ [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750] diff --git a/src/neural_networks/bound_tightening_new.jl b/src/neural_networks/bound_tightening_new.jl index 0e59848..1d21a21 100644 --- a/src/neural_networks/bound_tightening_new.jl +++ b/src/neural_networks/bound_tightening_new.jl @@ -2,6 +2,7 @@ using BSON using Flux using JuMP using Gurobi +using SharedArrays function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}, environment; tighten_bounds=true) @@ -32,14 +33,14 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect for layer in 1:K # hidden layers to output layer - second layer and up - ub_x = fill(1000.0, length(neurons(layer))) - ub_s = fill(1000.0, length(neurons(layer))) - + ub_x = fill(1000.0, length(neurons(layer))) |> SharedArray + ub_s = fill(1000.0, length(neurons(layer))) |> SharedArray + # TODO: For parallelization the model must be copied for each neuron in a new layer to prevent data races - for neuron in 1:neuron_count[layer] + @distributed for neuron in 1:neuron_count[layer] if tighten_bounds ub_x[neuron], ub_s[neuron] = calculate_bounds(jump_model, layer, neuron, W, b, neurons) end - end + end for neuron in 1:neuron_count[layer] @@ -61,7 +62,7 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect @constraint(jump_model, [neuron = neurons(layer)], s[layer, neuron] <= ub_s[neuron]) end - return jump_model, bounds_x, bounds_s, opt_time + return jump_model, bounds_x, bounds_s end function calculate_bounds(model::JuMP.Model, layer, neuron, W, b, neurons) From 6d092f7c8958951e7d37409704768cd19758333c Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Wed, 24 Jan 2024 18:06:41 +0200 Subject: [PATCH 06/32] improved distributed bound tightening --- src/neural_networks/NN_test.jl | 13 ++++++++----- src/neural_networks/bound_tightening_new.jl | 21 +++++++++++++++------ 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/neural_networks/NN_test.jl b/src/neural_networks/NN_test.jl index e2f0368..a5dae11 100644 --- a/src/neural_networks/NN_test.jl +++ b/src/neural_networks/NN_test.jl @@ -1,9 +1,9 @@ using Distributed -addprocs(2, exeflags=["--project"]) +addprocs(2) @everywhere include("bound_tightening_new.jl") -#BSON.@load string(@__DIR__)*"/NN_paraboloid.bson" model +BSON.@load string(@__DIR__)*"/NN_paraboloid.bson" model init_ub = [1.0, 1.0] init_lb = [-1.0, -1.0] @@ -18,16 +18,19 @@ model = Chain( Dense(50 => 1, relu) ) -const ENV = Gurobi.Env(); +@everywhere ENV = [Gurobi.Env() for i in 1:nprocs()]; -@time jump_model, bounds_x, bounds_s = NN_to_MIP(model, init_ub, init_lb, ENV; tighten_bounds=true); +ENV = [Gurobi.Env() for i in 1:nprocs()]; +include("bound_tightening_new.jl") + +@time jump_model, bounds_x, bounds_s = NN_to_MIP(model, init_ub, init_lb; tighten_bounds=true, distributed=true); jump_model bounds_x bounds_s @time [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750]; vec(model(x_train)) ≈ [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750] -@time jump_model, bounds_x, bounds_s = NN_to_MIP(model, init_ub, init_lb, ENV; tighten_bounds=false); +@time jump_model, bounds_x, bounds_s = NN_to_MIP(model, init_ub, init_lb; tighten_bounds=false); jump_model bounds_x bounds_s diff --git a/src/neural_networks/bound_tightening_new.jl b/src/neural_networks/bound_tightening_new.jl index 1d21a21..4648861 100644 --- a/src/neural_networks/bound_tightening_new.jl +++ b/src/neural_networks/bound_tightening_new.jl @@ -3,8 +3,9 @@ using Flux using JuMP using Gurobi using SharedArrays +using Distributed -function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}, environment; tighten_bounds=true) +function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}; tighten_bounds=true, distributed=false) K = length(NN_model) # number of layers (input layer not included) W = [Flux.params(NN_model)[2*k-1] for k in 1:K] @@ -18,7 +19,7 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect # build model up to second layer jump_model = Model() - set_optimizer(jump_model, () -> Gurobi.Optimizer(environment)) + set_optimizer(jump_model, () -> Gurobi.Optimizer(ENV[myid()])) set_silent(jump_model) @variable(jump_model, x[layer = 0:K, neurons(layer)]) @@ -38,8 +39,16 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect # TODO: For parallelization the model must be copied for each neuron in a new layer to prevent data races - @distributed for neuron in 1:neuron_count[layer] - if tighten_bounds ub_x[neuron], ub_s[neuron] = calculate_bounds(jump_model, layer, neuron, W, b, neurons) end + if tighten_bounds + if distributed + @sync @distributed for neuron in 1:neuron_count[layer] + ub_x[neuron], ub_s[neuron] = calculate_bounds_copy(jump_model, layer, neuron, W, b, neurons) + end + else + for neuron in 1:neuron_count[layer] + ub_x[neuron], ub_s[neuron] = calculate_bounds(jump_model, layer, neuron, W, b, neurons) + end + end end for neuron in 1:neuron_count[layer] @@ -103,10 +112,10 @@ function calculate_bounds(model::JuMP.Model, layer, neuron, W, b, neurons) return ub_x, ub_s end -function calculate_bounds_copy(input_model::JuMP.Model, layer, neuron, W, b, neurons, environment) +function calculate_bounds_copy(input_model::JuMP.Model, layer, neuron, W, b, neurons) model = copy(input_model) - set_optimizer(model, () -> Gurobi.Optimizer(environment)) + set_optimizer(model, () -> Gurobi.Optimizer(ENV[myid()])) set_silent(model) x = model[:x] From b88b74a6edb1bfbdad387902bb8412e3737c3e92 Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Thu, 25 Jan 2024 14:14:41 +0200 Subject: [PATCH 07/32] testing bound tightening --- src/neural_networks/NN_large.bson | Bin 0 -> 39253 bytes src/neural_networks/NN_medium.bson | Bin 0 -> 11308 bytes src/neural_networks/NN_test.jl | 19 +++---- src/neural_networks/NN_test.txt | 56 ++++++++++++++++++++ src/neural_networks/bound_tightening_new.jl | 16 ++++-- 5 files changed, 75 insertions(+), 16 deletions(-) create mode 100644 src/neural_networks/NN_large.bson create mode 100644 src/neural_networks/NN_medium.bson create mode 100644 src/neural_networks/NN_test.txt diff --git a/src/neural_networks/NN_large.bson b/src/neural_networks/NN_large.bson new file mode 100644 index 0000000000000000000000000000000000000000..45166e9b41b142505640ee1830fbe4619ca5e5cb GIT binary patch literal 39253 zcmeFYcUV=;vo5&FL2?$5EJy|s5Li_pQ9zU+K`^6Y!i1Ouf)T{5DCYEGz=#o6m0%V` zP)wj0Q4|x30VA`&Uzq1SbM8HV&D=ZBndv|FO1)Qi*Q%=Se(SCHO%jP*`os|<$4eqt zNhGpUhowoBBoax+)G3*%Qzdd!XHFU^F_-_>6BX%`5yPeq`^R1RNyDZLOV5xzl#{-~ zO>$d$_{ZP>hm}_Uv(gjk3;ySowo0E$-xU5|R#K2YmB>#RmM-nr-xb{?iqb>hzT?LX zm&m&R%Xh~Jt$@A$?##R0-?jeL zq?zeT{}4U?+Yj+St8>uA zDI@=_&iykdjsIJn|8uSXsPsXjhmD!=ccs5wf~3>(e;BWv^zaY$|Jy33{^LFWeEvG` zAJ2dO+fDo@FY`qDhyTuo^G`ASko|`kvSj{)n1)T6GHm9*d~L*kDTV*w{4D;i_)kyt z?@#x)j|&|?aoALk?$R^)j4`uDN=&3@I&PA`{np<_$6X@rz(0$^KfYV*-xr^^F2m0= zHd0COa+X;QXtVz~@~;hGzw9{AyS0&fmTPja(#2Tp-Vd*@Sj;9yRgi4DQ@o{lm6}_M z`D^-LLPx?vl698|4&W*fASM(NgMhPpFr|YxzG4p?*H40bkdtBZTuhO|4)|qf7=B7 zPnG|N(n=eX%3RLlf7a%cfA{9U>9qMzQOHUSQ@X;LC;x~au%$K>6(XxZ#;Sr09~=a1m?YX4f1#wnD4gyT$%3Z=>Yn_YpDqY+G< zFsXgGkfE!=xqj`0jhBQBmmXZB;>x=DeXx&}Jm+*BP7lO2WN`lvwc6Nl&y9-Q;2G%K z=Xed}Kk%W=`u4cnqDHhR2;`?;XK7fZA)e}b7$P;5aoN2NnDasrzpcL@#B@?;hxKQ~ z2Lnv;d(Cv1-H7ZNyBR|7KBm~#jj$x!54(<1M8nO-@X_Wtn4W7P+ao48=+%3_((Yy$ zv2!t;nq`BE4J)C%EfAaQJ=tbk2)fuC@lfY?!qwDIFmS57@Qvm9z6tL|%ED<7T|Dp`I5PlW%sGd%? zM&ryj4eaxUXzl_#Y%uDA+t-BRpwJ#XBRmQ7f6o$J?RLPlVVi~7x9*Zd%2IJ$Lo20> zP-M$7D`?Dsg(MT*2?w0j=00C%!pZi#@cH9IQ2VZoWuCj>=)(k%RZQmevE5Kh#tUQp z+%eHG3C6tGLb^})(_E?}t#j)IjCWw|Ua7*dla5@P<&UNI%A|6xj0#7{z`k?pywvrh zuxHIVs1I<&rIWnbq|k>y2xjc3Sx=sJ5&XN4HhKmrAv{#zSzois+GdGZZ+QVWw`uW6 z<0ImU_O}#X8h`;=!(a&P6K8+hS@vqrE^(Zv36H&gRLE@FMV{$9=^9H_yJBaU zd44@ydAQl{`bTf6tScz+X_4q?{2R`XGNChnb zw6nsIlatsjYCmZO#N+tRk74SRJV?7dfR{|{gWkOzP*c|u)?Ew54Vz;4LQ*#{-7DcY zt_ds~asw7>929+Ke*)*iFq}JODRuhdfN3u7aJf2!AB4%^*(D*IoU{mjE&Bo5@7F@# z2YR@uPZzxY=MNon(ZXoW{@f6l2Oie)c<_{+u(%iU`A;>Fb5IX`*JRM~GZB0<*PA!E zf2BEnmcZDHYWS>^9oLtQfOUn}{TvoNfSWfHID42oyfcyEK)e7!o7&)FqZLj$6wMPB zlV5mC46i+@z#pJFu)w zKQtU{ghsQ1sc3={ZjMjJZ^PfgNEK7giB^L#%^HHqCQtaSZG}DE)&Qh`5L;6(Lcb3_ zIC|0virHx^W*47?3f~W6ZJ{=sSa!lAc17^6{sr0JO2nbV-FbQ9C6INzPd7)E!^LSj zbneYN*rC`TA1JS&G_wGf%+bN7uJZVG@pzK-TL~%}w&?ir67}8DPq@;5BYpa2BV;$M z7m5|*xOCoPaJpxLGrq5&$_Ksq@-%HOS*47%F3G~)gSTn#`x1g?dxT!gC?%_!;+xJ< zx^^-fFZc?BXLrHpeZq0J(l}VTPLV@hchUWEeb^u@h;4l(7{O7z@24>z{{D!XHrS#@ zdzBa#dzBK49I)Bq?I))E88BSdX7{ zvtk{?5@FgZ4a^xCj?2BK!anyGu)*Q5Fn8%1c-?SE7&z8KxE;I^HYj%IH{Xc5jrHOC z8-m!rV*xxmb`geXPXLY1UrA&4XIK>d3O1~N;48m-q4<5UIXCt-Va)v%REj)T{8qrC`Dh%i3{#^)Kf-%{aqQ)O_} z={QapdxJ*2Zlu{S6EL;rF9?5h3|5S+gKY8`~_lLCm2hKh@4|oYEx2y2Syw??`X9A1#kt zeYM9DqB;I~sCt3``i z#X2#@(wFLU7sBYN6QKL|7<%6HnM}XNa_(kj&dSzewIk~2&_jbg@i#rY{2c6O_2651 zx5QI{x8cz|MLaOfk)ORiN8`rGV)Aiy?(5N;_eDGSZFzKB9C0OwjLKKQ#UEDaIX({i z@0~7c=586osSqYz|4glaJn8V3 zMp$xotzaK1ig#9S^xN5Siy}WQhf`hD`Ps^F{Bp8_rj2rh9igkqQB>nIYoRj+F5rchxQDvJ3Uu|^9!>>Znxa278vsq48oL_=w)L7W3-IJ#lTVvBGrtY`P zU=KOr;3Q{y+V+PG=J&zNy_5M_rVZBQ*ppmEEnV}x0Gm24fbH&heljN?6k1MGh^ro( z{WwAg$2g&rgFUzmT1M9&-V@}8_UFSDC!qR3544&1UF_TK6!mBi$G|FYPP?#B*y&-* zbs_oi=V~M_>SD<_LIvjVzxM7j#nkRA^X>749;eq(>S_bQx=IUO z>*m1>wHgRGql>v{gb%#C@OQf~EKvshyj2HPcBf!n_m4E=-3K~!L7n6MP1){PKb}`+ z02l5p2AOB~=ubic>2fdJ{eFvJHSsDfcx28eSN7)f1MgFAqb(kNoG#9d3Zb3Oh4kW_ zH&*cj$}=_Qi{*}-@Ky=7pE?F*D^7#=o=rm1+E|{{V8$-KF;uo*moy7QaOYANzB%VP zSZJJqpR#jl`~-Wf?0y8=%HGrGF&eCv)r&$__r*_c18{rUW17G532c^`MJYqylIt5C z?DF9wq&RqrsZC+5tGiOXxVTa1TIWN9JH3KkH}})QxzRXT!5Q}#_2j5I$=J7_1s|Tg z6Ha<=qoI3@XxXn&toq`FuRj{%3D;+k|7nykDAFG-Ve$}xWAHAhX&)MUzVs}drM4n)esNQT1Ff_ zgm%kCjZ(?SED=I(`~+DfQmmpYT~X^K53&#uQ;h_hE2;*rsYxbD*e@pM!) zM<4IbwL7xK4b>m%ynY<^r6gQi6N~ej7SrP#2P_<0O*4w4(EHO2S}@=>{7IByGwEJL zAt_Y+svZySsEz@t@@UjiPcAZZ#OSL5bXKi`;&%T8y}CE#<$4Z=dnT~@`u&2tjv<=G zXVKQ<3G7KhDC>O$M*MmNfiG)d@e3n(lCYP=NiJx4${5?{EvKOKsnC+;jopt+{YSQR zPLj-`#r-zJhz{A5-)$2lTRe4zHT}@=r@O&S>|Al;iD! z;;;);ezz+}jrcA8@;e0K;Ykqg*_(R|RA)WqJ{%XRgcJHmfL~96h4qi)E}2_lCFu|!r{>TQ-aq1JbG?xil3|l(f3;bsT&z!`hG_&O;A7=bq$Qz zWePL618OV6@qPExH1w4N-$8v+uPnv04 zNN-@4SXdT8#cM~9owhTdd3K2o@AZL~G2KvkJBUF(8{o!gcTngmhtGemq>7vnmiO=_ zt=J3DJ=Krwx~`^~Ka}vKY%jLDagTBaZ-Ye-+hCZS3U+o{OkdrjNxR!eSeapfr4Jp- z8rpjD6ms;%&~;?|vnRFYwNmM{PvWmQU#@DF;jobbY~1fL;QEWA>c|4PyETfJFX_ci zH5NP~#fE!$=tFg8H0zmX!=9cpFlmcDH>jlGjN@G~LdPF7@;?iYt;WLmr=!oV)M$ez zJAC1WK`*{wX3m3bI^g<~-(|ZjY)hB-ze5H#cW7_dXRy=mDd-;e0H4F#c}xgz{u9MldMM)j@*b@JAPk2uxCP#A{I2v29%2L84^!r4wWq~p%YyOr)i&Yluft;X$N{_| z-H4aZxklG}hH$4n@*I(^2ju~O385<8=w{AYFxQ-V}+D^ z--dHW1!Bn@AHH_#J}J+$rD0E_Fukk*hV*?#EvFTU`#Io+Cl>Uf!x#IUj`NET{8;t~ zZ9(U_3kGhU2Gzl)Y~ZfNpDiv!|MfPQ`mzWb%GzLaQZQ^S|4oy(Xkw9XF==xfS?p;B z8`D!X_`@S{MZa9Qc)t_A7;M7Nc6xxqOa)k))L+ci$@EK2*-m@*4nWx(n)qjh84s;K z0Tt^^V5O%!X4amAmVw*Ib)78Abm@gzsu$t%l;;$XvR7Q~HnvRdl`(7X_ZIixy-#a? zuO^+RhFGGdB0{i$Gd?%LrdBz+Hrf*Hc1H1;dkMH+F$L!|NbrHD8NHuYN|(kTfctJC zT;6*G9IanLqw=kJXXX|-v!s?n6=P}Y^d<1^qQDvX8my6}$2;2>L9A{rq@CywmOpy( zug5d#gVoQns!!@Tt9Awa3{4hJ8P|iPz@2VrGIXfDh8#vY7L>$i@Q=kc^95{emXqguEbA!ShLZd&Zs@p3NMd)K$zI>HMLTZjM zW&e8WyCI&PzlLJ1h@3yq5i~{?!-VS36q22YDH`2z#<6kq5&Ph&yU9G^x+71P`hY?4 zKZO%5y68Q~4pjU@Ff6dU&mi@7*w9ywtM=+*m8UJ!mksdZfFE~Lu7H&BOdTCYJbsJ} zW-c;DmC|wI_#j2l?|4af-g;q@oju0hiG{=cl1T5*9kB1{fRN6;&@3>V7p`*SrG~O# zuPBGZ#=ipEzK;?krF#bD9$fg_OS<;W5ex495%wt>;|>VqZLKfiRLUEuE(l@M-CEob z6M**iC&+PAKW6JUf6SH^@3+JZnKg9jdKk|1TSDO;Nf>_j zBPmEUdBYQ1HZa;vZH^Wo8#0sT>4!rOrNXZKx1`^83U=1n;*aqX zQQx;W*!S|mjD>A}qlY*8t&an2RMAAkDaq`U5`p38En!~!1-f#1GHeXG45M| z8w^$|u_%{2RoM8KGi>$^!NYfb`9>CzaQtl59JGA_20Kd=-$K+lN4^K^{&s(3tfu1UuIIkVn$25Y* zj5E+YG7CI3tXQF11{a>u<}FgZb5>hd4mo<7rl#hBPSEx;y?_CHQdg6`%u|FXzn9Wi zTn;+PPJC?IY|0-07ShBvNkgDI?9V1U!-euUTOo`Uh$ zBk<|7r|{<58Q6BbHy<(W4ArkXLFL5GxU*|dd}wCOp_R*E%>3V!eJ2Dv_IGA4H!FS} z+76;PlDc$CfYGxwP&Y#zPN%OSwPH&ea5S9Gw{&K=U%5g=Kja>8i=560ob$s3I}{px z3(9?Q&6lt?>_r@_2FEkeZ6QsH6YEsERMkCQhqqKnS^ps;K> zS)U2v#tB;VWA_x=c<&vooBU8v{;`BgH@p|DU00CPrd^cawj0)J$)-+8Z=;`EPG=HddePCk^6df4zW4=-NsIa!$5 zshJK>jKW`qUHMVV0TS))ap@W*R>*#zgVkd0WJtGGVx8OA5UBD324v zl6YNRH!OU5ms*>TLWZwCPg!J4F5enS)h>oFyO?nN`Cfc~*cos=<;#&f6L3n1I~Xaw zpxH+1Jn{Bku+Ldcd1cO&e=L-`Y&<21#v$Of$%IE_T6594zO4U-Y3h?g3UhG8)_@E$ z2%1lio8+C*oSAjiS3&6!&Y6hMq(5dP7*%U>$0ZY4b@je_;5oUC?aifZmo$Y#yho&p}=ErTN_+LG9IN;>=$tS+<`zBp3SMk4HBh6L5+j0{TvGfR7omOwBcVy38I&j+9~Xbv*kmT|{09 zI((wnJ9t^$gHBO2_BmTGJ`WF~6HAvvwuKJVdqi{bkmIl;;a$&w#EWozM+}EN^yRPq zekh##B-*SBLm9i*5IFmTxV6xXDmn$@pxiF#U}A+kmucgr8v*Q*RYS2=o5g`olzk%Y zwYW;v8C}!V*kav8@^6<=veE;(DgO{o#7EQP&7IkHTPOz%X(gN9U#ai7^N`!o6^A50 z0gue5BS$mmB%~>(!)LKw=3})@0MyQZ%fxq0E#HY21ynM}CS~Q^*Op><3;@mC5X8V0* z1C`%{%9=&sFs?ga*6)PjpZk+xbr28#vIu6py-sGln9}FI1qu$}ZQ7G4bj1PKzt$BG z2Bz?az5%S~Gflv!_Sj{a8tPZ6^Y(-#!m|n=RyGRY&E0=N^r+o}K0gqzt2y%6y@v(W zbvFFxdMhQI_T=uHKGB7Ts`z}AGRr?!!j+*nY0s!|PIYV-T-7f^LhC2E^YSpfGW(m4tM~$F8QBF<+hyiVe_KB@xGUY!=2F ze-P8=^uVoOjd6R1K002E!+hx;ZPsTA8g`B1+1O6CgRL_fWrxKcFxv ziZ$F#uy}10cCu9GzwTY5VS*DrNnQkA%XP8qjYiT>@5|$^#bWR#DK64K0?*A7_?b=* z@b4eT%huNl*RDOL*?+!3_NY(7%*ipF|8W4jK39dWO-4L<@KQ<=FM*D94Rl3*0It@Z z2@O3$*~=Di^kGLXR*i-u2bK8aB?U}8A&-V8DHt0UfoXLPC>&LUZ5O)0&~5=-eKnYl z2aFWg&eDR!p^sqC*UY0TqQ4B%ODS2TaIv22cAHvCR@;olyW&~5n}s2y&>k0W=2 zlX7<1Ut`|V=xK%+lJJ|x%+=v1B;!V%v5ZBF|EjFzsrNI8E ztUIOC0YtpxvcA2B%B?(<)OQm{vg1hnJI0y&5j|Orn}A zgTzkD=g`PnH}=V{pu)KCFy%!I?zc>2k1LLRe}XEWwD!i{k6XlmJgF}p;*15c`WSVy z3s3jgDm}Mo5Lu7Sq~LxLoVzLzu-*)NDm|s<(>uYpPpsc8!(J4ZF2z~3w}QnsS-&NB zUO__CGeI7EqL#`a%J*-Du%#yaa(Oae8{Y&ea<$a-`wMudd>3~I9~D;ajl}dw1wOXb zpWY1mEKWOYL36%77NX7ilkxbk^!$lEe&6ehR%x=>wMKbP;vCSP3x~ zw+TzMw1o6SeR%5+7`+7!0CV#-BM z6QH`v4d2XcfnyhgghdmRalx1gLdeS)R#@oEx91ph#bygEvakXD*b=a~`z>i>}7pk`z3Qhn?9x2Yg4|9H;-+42Zlw8JhP`8jlP!yMPP(^@@_olY#{d-Cc_@x zHK`=VoSSFba(@d|=+?w=x_<&YE$oaBPYW1x)`^=WCqUc73I2EmDjR+g`4|1XX@-@JalEiGhia0JLAN-2PM$47AGfbm)AOR= zNQ)zIV!{Jb*0+UA-%L0v#*ocyOgLZy(dZ(2p(yTI}^R|lUfW4yBY*f6II-= zIe-V{Cqc(q3BFhw2df@w-~r87l(b+lEZd}x0|%TH-#jg+s*@J1{jf8S_$-h8?e~J* z84aE}S%KfI%@AIM^#!>hd8GUN4i)HK5r+#&IB_fn_^r&Fb|++nfgwJFAf-bw=#;sgh!AO5wQE7IFK!-Owg) z&ih7(VJo-8y2F*=7wduFGSxU?d<=dZkpP=Ya!JwTi%_Wg8NM_fp#k^3gp|`EEcvhy zc2smh$>UITxn;?#9{r>xTKC9&KnFC$>GSjVZIC%36knAo@P@UD*rs|C#{Bs~vJ;)5 zS>_rz%_6wjaa@$snF%MN9tleq-Gijmg|umlKX)D03aOjTnab_hhWRQ9AsT@SQ8 zsL8J^$HAB*dE)lbu{^+9I>%|qkRn<0vI;|3amRq~r+V`I&3%`drHq!_(#!ns^?I<~`=c6({!p@~{71iEuS3pIE(NV;CHZlsJL4cw^Gi=NUB zGKw(AQ7^jU<v$dd-jW~vxkG@n);a;L>65VG(iF;R# z7H4Y2asBlq*!4hJ-1N~2ruTN?9Z$lU4usRTVgu;rsU;2>lnT|#>5y=?6V6vSR5tHJ zC)|C(AI{$2MbQ)4&pWJ@jNiV7g7Z_P`%q&}I&hMzE5E|A6PL*LXCTVD6@j?+p->{l zGGn^DBd}{H6O}d4DSR@i1~?0&UY;X`ZpePaF2f;b1FT5X!NU?77B3n^@!@Pc@y^%I z+^hIJ=`Ni|Yrb@1Z&L}{`b%xJwhmI>KBEsXT3p;`2-xrGg(=frSY`A);lQ))r2F^= z_?}E+nQ6PoY_<;?4V9x%%TDyQQ@=9RrLX8$o;=T-ZjbKXCb*@)E|<#}lqt@62&VUn z%9h>P4jy|Fafqr6`?YlC58EF@m;NirWZ`Sk?W_#8qzt8|PC0b;sW#ux{z&uPJH(5q zfO(w*aL>00UM`Je|Jh@Q9rsp%m&Om;xOF2H^-I8_%hSYBy?^=0Jk-EO8y#HSp2T7A z6Cr%CJ2iV|K=gJgc9wh@@}sh8olI|fut*J`KJAVV`WoVu@<5(qk?rSS3OFsKiZ)C- z4R$8Bc(<+_TwAWd%TI=KLWeP{dI>zs=PJdO=F<6(vBIE6v^@PxgY>o3Tg2iz7v7#|b+Z>xb*y(Yq(YZJ&@*@>4bwUWjLOOT5mEaaF;xnR?Lc+%|! zO6{jD+C+6QA#1+d%H6ukc<5hgx7K(}m4C@0?w-_}Yw3f-c4 zY^o*S>lTQ64G%(e!dv<}+mXlgjN^OS`^yI2(8A(xpW#QRPUP@cJ=`e^2HmlU8h#t; zzdQ0OtM+_m7;gvni87sqvFJ@Dt--@z<975`6w0KJ=Q?lB2lP<5n zLJyuwv5HH_g&5Z+@lN$mcpGz;Oy&h)wW}STj!)t@yOdyRuN;d0t6H3g(s-_d0dxtH zNHJeSe0>P{c%2&M4wUW#?l+4&(ySoprvvSpz7X6t2V&pF^0@n^yr6b_GilqG!cprM zI6X6(^mLy?(FQ3ll=VRL3Co~8b>?VobBp?Jx5j0+=7=-e8=>ZEGWNdx3w9UF;Ot{k z?%VZ?)Jt1iv|IKG_n&&JW*Rs zeV?oqlHXN<;jjLD+4h%kK{*nGzITN=mg<;Ux{SU-jGtvz8@bN4#uJ4xT=V!P1s;tC z)n7sE*!c-uyAXimex``EcMC}`)F0*4o9M`U3590Z^N_9i;@K63e7nb6D1SGQdi=GR z9$2-~w>xjibWR6ta{egB6#LTDs|w^@`A5*Z=*ZbyE%8fJI9sg>5U0D?mG+C&q~$eH z=r`Y1Oz-cb@jxkEh2<9uWIZ6YZx=CeM4m zeEFLPZatdFD^+`8!lp#g)38m{iQWX&mh})}dX#>dKZ2oyZ}|30{|Ya<$8h{L7yQ&O z1aV^zPTSrSL#_I950}+q+OqM`EAu_nCoCtiBoFE?lu*)QH}=o16dyzboL0>Pzwv%N z6Ra>wCXe#m)G>LM2PxwQ>ay6FH+9$G0$nru?3fO(4_l#Ny_Y8J?TLr%Pr;j&GB{S{ z4}ALY0(9~{c>2*Duw=Isce%VCp1qNB;-ZwfXKn;ujwpbJNy|XtWg|(~U&0SzNSRM# zIDlrbIDCRTD`&;Rx+QPLwb8YHJuGc`nV&IdUopby*MD)5 zC`R6I0oeR@0ivN%&&5ET7#I zfh36$s>Tiz!Y&z;jTH0Gl&hfPiam5EUIv$ps)3VFa|N?tO$-zMBG=VjC^1zU<9gT% zdVO`cvdE36OL3W$yC>lC!#dCmGs61OCMfJFqoMo1L(9Tih^c%{AFa=W`Rh5bz1{?0 zeYS$xIo3?kv7EQ^1jW^Ov9ejF5Et)>n|tr1AJeA6wAOc0d$r|bhX>%OSsTcGfD2zT zlX9jsmr!Nmd|JAqp0xUga$=kfcGj12Tjx&|CTD%1s>{~+Ug0Fo3UtJ2gXF-=zz_Fa zlKQI$8d%Nf(W6Iy3BxD&u+z+Ul&RO7E;%2CRV0l=RL`Uk)qWV!?=O&| z0K&IrvHYb^08VP%Dt3-tLaRqTq)T}^Y&S-Np??(1PINm&&o`ZhoIAJ0AF$ZQ;e)>Bc~%-lw)L$Bd-U*r03m4uWS9q%boPt_HzeR*vsK2 zmj=IsZ5|kWzZ;IX$fW22W#X6%JN#_=sbQxUIiT>~RK#km`6v$*o(9q~Jtb7y5XA-M zefX_;G@h$5;`$*^;LgQybY*5KoviD@FT#RRLAM8-DD>y5g&thbyK*GJYkDDXtL1e{KQu=Y2WuWDu@@)(Riy$zVuff6!gM zQqal>6E+l07n;5-6u0#L;P>uYAl=E)pcCfP#n_yA(m2w7$QRawf~qF(AAAwUoy>N1N=_g4B%QN7$1wpeR5IU_w^V09UO;g10*O@ zepQ%fsI{L?7*zUHGtj6@`m(qB<@ewI^&?pSra!-ii zwFaoNsFEh;dGq^s?Q~=4ZfZ#h;(LzEp=xXtdMx?_Px%1IX9nTzDhpiMbcP=Fz9=l( zn#>2K+))<~BlI8RgRe6FkhW7Ev=&dKU(&evkAdd6ILsIWmJA{n!z(m#l^GaYgrlZb zV(FXV7yYi~+H$u{d3-Wp7>u1-1-*6iV2a{5aO>s>D`Zq5>rxcbhv)F9;e_~W$UZWy z+X*}IxVCM(#x%&ZqO?J$p1y# z&Aa31frz?l))-Zg3f_uZoSvkHiP{G2;an*(TeZ-0jvTK% zE@j%gS+Z~EWU8AS3_pJChDlmGDI?vFzx`|hyV7)!RPPg?nRe%u*oS~e$%kVs^ zGal;Z&2Dcb*d^~XB^%zM(v_>pZs&VQdmbt1>)LT#dnDxeZK8&94XTze_8Uvj;mU(6 z(ivTbqUsB!sO7@4ILK_3My_l%%)JsCW7L4ZS=7I@%?I@;+cfVy<&xw77ZYsLiO zI|~=4amrZP@fIW&T4LL$A&`3Aiu=hq^P72*+$v?r?wDYSC&K}atb@_`s;SWCnokX% zj)*_z>yvY|7OhtZ=cu`9;!5XRGZH+hID{#OcAds)vmx?-WLABL^GTry{ zVXcpn6k+!Riw~hRt|S-_)kvr%qZ?ir(g?nLx^cjk)3jCEe)HZzVu05OkciI(^+rVm zvtC@}^NF;i?5fNKPHZQ$5DLwsM2qz?wDguDX2C@&{9bo9Yu;Wslya7u+a5y8HhHde z=)%8WRf1z2VjX%zopTzMtRyj0%Mrt5d|_ADKD^_Z2`f3|LBPoG6x2A}Gb{{UuD;CCTxno%JQjnWeOp{u?d5~oUr%OIVL(ol%{Gy5GUjxv1 zLr?lPMIGyUwbJhT>lC!}IOU%o47(GmNb>S)S&79>A#K*oGKJTH{N(v9xN79ZFioBF z4;731Mk`~~u+9+Wd`moX)0nej^l7-23?E@Vt_xSdmiucVOMWQqcpneDx0-U?qW%!* z)B~H#L&ehf##r*_f}n5&xL~v^W^Hj6vMi5K*3(cb>c5)GU8EU62A5%y!!dFV4aK5O zZ@_Mx4{O>+qLzCL$$bds>hC6$HC_i5>@@gvFB?w&o-MT4A&xB5DTPT08KjO|toqG(AW>>lWdbvkk!^27m^`kkWb zk>4m!%FOaecwBbnaSIHsH$;n#%CN)I7_2{L!2@F{8u?4gj(nX89>FIlZ0ZTfk@ow} zb{Xzg&|8{Cpu|c)J0RlPSu!#-!0PO|W$L=^bg1PCw8bl8M7Ds&d&UZKhk~%APzA=` zErRA7eM!?CSwY(YC4Cb?{h%LRF}n$AGmrWmSX2YXQ!jv#vIUn03N9lKq&Z$uF9dnjeGs;hBH#A#}%nhfXcfISW-32h3`#Oh{U zYN_z!tY=bY!TlaAS#}0aAC_R)`5TbBu7=hQiJ_q*&G3r05q;aPf$En_ga|XljDc?e z)bpV9fdI)v&B!ATe6K9u1`Vb@6s12+aICiC!pt9}t{VnF%=~F@HehniDk0TE70)H; zpvS>l-_nFIGG22}Y`FdoBsOcH)S8eNzDg)Lc2_uH&_>l8D~0B7S>g{zS4{Wm3Z?Is)6vznbZ>M7X7BLD zd`(NxZHNJ_c!5ja-K4qFG#*LfAgGDc#OfJ(+&FFr^_=2|g*|(TLnqCH*#1sCSPb$S#mG zw55K%QObhJ=>|Cq8t8ds2N|zO2GztnbZAK!m+5(NR!|J&kM1DzR0AxO^%B>18%0&Y zCfH!63LcIF%BH_J#lkSRvUjwJR^B$mF8MO7UMFRC4RdG7$`IlAvv73oqYUZuyW`fv zPoxuVkEgG_CiS!vLgDcs;kceLFTEwlSN11E^GjJF|Arbpv0MkGjk0KL-I@-CMlfeRgOQ+D4q%^8gbHJoLdCa@#k5;Sn*!z1G zRHt?oqQXKb^P2`AxZ;F4qg656t}_R0lcQCeJE(5jc-URlOv$U(QBjj9NCLW{WN}Z- zxtK>qzZMCTUR;MX%h8k{R3#KXKQGF@vS;glJy};x10BlqXef1|246&rXUl1bL2rKY z=?2M;)Iz(&5GdTc%&&0y6mV8M2y2>@xyH(Y!+?i_^K0o~DKx(;PY zGr|r?ZjwjFHnFH|t&l!W2X1Y!M&n6}(2&*(l&mtK+1v%KtfcJrRSppJrZWfWyrOqD zvmw{Qk^AlF%Ie$RQsKGYBw35WMd}9bE-Sp#T(;xEH_DbW4h)wR z)0KDmQ0pwqRCojKEh&Qho2HcZZk=CA_C%^&^%{N~d*QBO%6Mo-F-0$YMyYqIXl>t4 z5G{??o!3^ts6}}caC;gZ3f@Fng-%d;$(7sc?OBp?hGvEAgE}cYK<=C~{+4FqtZ`7~ zS>ASJ*YC1WUUNVUNUIZzb~lR7t?m?&vzY>JyrQk2KSK5OUuAZw9?;Oq91F<_ZbkaB zo4f~KxKseMraM6OorQ4cnJhjYq=Gpn22$CnVBw}o;wtl8h&s7X%-OIDz8MI(a*#Vp zysijjnGc0$%}a}dyh(B*k^|QFqKs~fg@*H6gtVBoP;yxY4yZ`b&Z`E(pgZKpOoO_g ztEoB9gM5cAgf5j8khZXx-bu7@?oB;B@YfG;tE!^30b%eU!H}QqZlLDPzMLiXBN6MZ z`OrNn0`aMXwoZ${S-Br!=~*o-slFhz&Z&amEsH>Io|OHuy&6uQ*2HHXT~N8G4@Xqh z&=t=RHeYPPPbzbudiv@z^#~0%4(I_TK4-}HQZGzfqh4k)0{?1tyd6|>^smb3qgOEU|x0kFx~5_#|>Jy;Kvws@_5wZcj&=&%6R;Y znx&EWx!+EJtF;Wfnn``%h&6)Sr}xxmZ_8TOJh117PCQaGgrlEo;QG}*Y;5Pn)fO($ zoR|QYmMh~}{jF5eG+Ru&whfF90G99E4%LBksXEz>()ey!z<@|-Q#eEtonL+jW*B1i zt|xxO-#KGRO$S97dvWtBO=0MnQW~qg6Uwt6fbj=S{vaQWQByQo=ZqX?9WjO_=nmQE zEzm*V8z;JU=iGL6%#}#jNPlVItiau5wMmP&Ms?6C`yBdlDwvbYtZ~vu3Fl7O0d|F5 z_*=0Swp}cO-ww@`=G&+2%IaT`rL|Xxs*(}=<#_O3-B4+E`#~yPaDyx+s8MNq1yrxA zEGxPHfevao?QXx(kQGcHV=F*OuzRK)B^%&hjSzB_+>~-Uz_8sl`h+Iq&KWZEJUnFh`f$WG{6-FtAJ}0&nF`EylX2^DG0pV7X!u8# z18SUc^rGIFaWn{j%=IL@9hIQ8*%hN#wEQ3Cy?b1Y+y6H_Iw&!uP$nso>8Lak&Gq?o zk|=F$eI0g0gmKu>7CSp4#m;Gn5JN=loZ7Mn(OjP;Lc}I z1v0}dQ_(!t96c|U!1e+MOzvC?2h-0&*+CPGDZ3-Y+;d_{W{K4FzFIIewn2%|mU1^8 z1<8OzN^K9FpxhwCv%x+LHnzaQHFrT8dxfIUPKQXXUMNj&3kGo^C0&>aAwRW2$D)z6 zuh|5B@3dn&{=8lu*8>ygn?MZS6ctJA>@Mx0;n&IJDTd}n6sc|K2Mxf5o;+zGumDOi~C?@;3GMV<#Z8vNHx zaQm(TqHmv}HT470zw$N}Ea^{|TNTXy{VK@%x`s4PHz67CfclSS!C_Q5%dDME72VH3 zv&&X!nDH9679y^&cEFf@XGPV(8ql273Tf>U#X@NST92^9xKZ=Tb5bxkb{Pdu>8~M? z?_0urUXlO10dRiv05m>y0h(r23wbsQ_MvAdtY6rR#odjjPy2nDm${Nfv_>%XSDm1b zyr-;Nyd83Ok3jGJHh4eLjt%;`H!FN@f=&7PTv^ha3>nRIGPV!qsiuRmM-Iq-*O6+| zCD8B72b=H1P;%>ukmqT`GFNYgq-PsIQ`%1~whm(zlhQ$=EaK?TWzovsfyrMy6!iT~ zF)_Xw4i@{M{@`b&Wbrf5Fn>7NIRf<^uL-JE{e;MZxkAn%JrxL^9=-&m-*tlMZ@sB@L>;I% z8u6{xtwxls@`aa^t%A3--I#Aa1W;38^DUAo$-yXgkm1gAj4tq z!Zj;{ymdDq;mI{9ot7q?F9b|1{sx{C7E$&!bBwy+f(;wJndaSXoxHG6jDKv-;?~;G zONR)S_1mwIc(y;vADpJLbDdFh;fU`3t^g*9-46@%Jeb=YZ+t%44d=~rM3;+w*>-s# zmfZf8!eYOJg%9nS=ZpaI{uYdyw-bef;xbUGIy1FvPpEZng816sNVCC)zo(H_ck{vQ zp+@ASxks%z&!E2dWsn#$l?AW1iXk!QL8j$6$=!jF5buj|mV2RjMqf<7i>P>TN@#dK z9g29(v?kvM8^$JpFIO3-IXI*A7vgRH0MhVIEV6Q=5ZS|4D01$^8uDGR_exV-bax0- zZ>fgZZG)KRdcDqK$~BN$Or@lY$06i(F|TtEg zc~Nq6v$CM0F;h(YUP!gtPm;lTx`I`!1z8tM)K5;M{oZ!0>Ck<__~>xvf3P3qWY6vY zmG^rMQim|6sF#U;s(WO;c?8AuE*8p^4DNHcdX7^D#3pn{{dbA*#ZJ*wl{t$xZ9jrlMt9sZZ2-&J_<`3u<4?%A!m@l`A9V{+&I__fePIdIwe~|r`!!Uvqdj}pu^WrY7)Es-S7^c4AdGG)rV8^< zWHUVnv_mga_4YULY$A7N{MJN!b`N0H?k;$}({QF?K|+PwCh}YHjUwJFSnT%?V2yns zMpOi_svep2=#Bzg13rVnFHu<0nLBuVdXNRL{mY-kL&3;%RD0((**rN4vHMQK+zG=` z6F8SHS6gD_y`y56TmS3u$Z>ZLb$$5hH^Yf+Bplvl2toM?mmNzo^yrg z)tgY7wSvm*Z-ZvHtq^y+3aW$qvVzo$P~h91>KE6-utR>_(Hz7|0}(YtE6F(UGwF*K zkYefsuz8&U&Y|6ys={0+|Iv#rdg6(*W{Z&iix(E)3wl0gFqW^s1$9S@Kz`;ueT#R( zX8~sT&D#{cppA?wVdBs z7zJT(-$Rkak$EQjP>w}s%$`_5X}@vhXTxEk{Ioy#>6cR4v{l z7G-}cKkZU7-*br$~5DC6{Iuv312=cFwN==$^D=48pVgQ z{HNgsQdMz+!beyfN2OoN6j*Mo3tSHBr>*wisC;fMdKRfm1ff!m0`!U@SG=*SdMI_kkwpOnmz;e8#^$+<5V8v2+xq?z8@Zn z-miH2mu*MXTmm7rbU9@DnvtyYMp3?_KQ-kH;m_5MlDav$kT|vk>eVZ$q-``LSuTP= z=N{zdvo8cpRbb&!0L8Rp)bQd9eM@hT zslx^`=@?5^_*%{q8j8U}_#K+gtrpFE6igl8o8n3`=-YgODKx=?zsnieJfRz|-t~gg z{dvmLmwZw`|3a06GQiwxAlv6{gNG7LvDv1EGOl!C1s|LtVW1z$y6qACoz_BPU<*L~ z4TyGjAxX8hSl(tXXi|O?#vZMq6h~(kX9DChqzy~p3aYew+r$FuM7b*;Q4UYLN*dCG zG_#!H4deL8UO%jQ=)GbQwOfq7$0S;_TCFiiRj3J1vPqJHyaJbk8f9OHx4?-qlmxw|gy$Kg0jCSh=b zrySL9f(1P7r{$Rij2*d;Hf}RU+w#8nY+DF3d|Mv)Q+X;5!d5l#kma zPUH7hTY8TiQoG@CUx7^v?2Br#JuC0!gvqf@5V))iT)LPtPsJ3d+IkJtMVrJlWgiSY zUJps9T_AGbT8h8x!K(M=Kw|2BP|y4-%5DxOMU+mAymVhsFJC4!-Qj!kjYuJ%YoICP zDa{2>92<~LRbD1c67R-hxKlCvz;YR6Do$r=)UT z;nzID`)~Y^^M2UPdO z73CS;EbYMxj)lCY#Gt$6HbzI58@5xzxp>N69|U3Ff~e))Zj#rK0ol2fg`6L}|N_o$EvFi_9nZ+zW_Gaftkfk|6;KP}eF}NpF zpZg9R)4eHmM+Hf|&x#Fuo56G2KC-^EpQqoAA@BWzShddrs{T~LLr4P9%qf8P@kga< z{0XJ|2p^{D900L5u2GV87)o2DY-5u#KH)f6!t6w?ZZ)T;sf@}%rUmYm6gO0 zB=H2au}{=sX~Nwg7L^dUpa(vC)rplZ4W?AZ4G0`72ic;{f+8dvPWO>w?YA?O<#mq= zf_Ff;-&K%&oXPhRX*BPnoW1F~84NF7$-gX|;x0N<XQeP+#RaCIbC4(LDsCntdjJV zreOTJJEpm;r6lEBp|M>%mez@xl)9k}Smfa*9*A_|~QwAxzj`T*li;~KKI$f}5-d6iaRd*Nm zI1R(R3tT^Q&XiL7+yt{zez?mam}w%Ju66Zen(@$?RTi02L+MgTGI>k2Zr=#HwbIhb zgRnT|3)G*;0~>iOT)x87AU(FjNK-E+bu`0KD@`zN9_JlF6>t}leyX5+hr>|!*%VDLdN8NEpFmTwiu?nrXw4E=c7ELmbgW+m#e;IW z%hMS>FL+blx09fijsx!>yRoJnHw4Xy0A^g04eO5rw%BqUtHd82@kp4%rs%zG21z(@E6s5J^(UA__8j56&$fA{ev8( z=hSi3eosFpAIvewiJ{E#u?ZbK&$-1pP9&EPVg`u}(_RJ&hOD9FyVx65cNQrlH}m;- z&SUXVp)HI0>p5uTT?D;v2R4n@yEc>Rs4`&%wXPX}mdX2R+F}XH3hae(HK8oFvYgki zlW28F9@VUS3Mn_NK(W%A85Dz9(eo@wi?^YY!}U<%nGN~e6}r855SE|VNC~g|pn*Gm zlh(8fDGq;7Ld_#kb8K-9=LKbsdnsf5GSXJv5hFK^p}d7t$)Tq+CT*BPwr>#Y%r8RR zn3v?YXBntD?@_&IgOJ(R3nYIAf@sCsQi@vSykU)KsM|D8~@RZmvNwyef{Ii+-eL2B>KAb;pXszvUr{un8)t{a}olwPoa(-3`v4>WFp|Bk-jH(Q+#1 zqHK4Ij{V!AY*s%=j5JW5M`yDBw4E$==TZLNt5kQQ28xGVqvoG)LDbCQm^C{T-yANb zQ0G95ZPLTAJYSaf>K2q8vSQXBhA@Yoy_k1$PiEG28mvg^&*Delg-A1vkm>738H;UD zR{k$BQWYo^uImXQwXIY;Y#(^WK7(0%bIEgolF0^c0pEL}Xt6enkBxUav$oz?8E8ih zef>FB=p*FjiqNzvnaT@IS>{j4ps+Gwu`|<1XVM$fN_vQK%JY=qYKhf?C9cV^W5w^^ zlJvy^(C=RXsdibg>*+wIdeo@&&lMPr@#K!q23=CjOrB=z%UW#r!-9m~EV`Wo^ZxV< zT7C+nmtnk*ij66GyObwJ)`9G=Z$e4eB8qLi2i4r^Yr%QOjN^IG_{EjQ#gAZqJpDvp zu~Jlz$R&N8CzMLAg1XHxl+tiWFtG$p-Y)TEs1=KO|5`{n+l|@Gsv!N5N@b+zN0Ipx z#cSK0`I`2pa?*XCVEtk*#GQ*^olLlc;)M)l@?nB%R6FJ}6;QF>h~=oev1=2CU{dT3 zsJ>At+9#JnbfqQU{?`bmNqwZOZ#@l)U#3IroBgCVo(G1V#|87nHatBg3=OvYIejIE ztRgMck8@-SyZI&23CE#%%vwL2(&2O*(5 z-v@5It_%EXgfj~GK4i-#Xv*I~PG$9Ad%zFF9uH5L5jKXwx!E-eQSGb;}^F(_kvR{XLj3N8B9N z3m~$aREmcJ z#cw8kPK0p3f+reJi-WXhp-?cP2L^hr5G3P!2-PO};=V*LRz4vFrAInqnae;{dEga= zJm+ig%^=eLwg}XGPwx4!J4_qm%ATZ#u&5LK`Ss9(s&~51X#a4g8vG85j0a$Wq&-?+ ziGbY61ZKya=Ug0sNesB;@X9Z)zOGbCcWA`4m~heihMJ0=Y$4g_0@2WY3pqAiB;&hU${Fg8@x6YhjNujZsOM0O za$w|d zZ;O~%+lEPnBZ6M*Bj%X}qx$4yn0-aUbztYkMeF%I@bhCiHJTa~f z=YT$+qS_oe#C-Q!P_YD1OpYeW&TB$;Ydc78R|V-My-?*Aplj&c5gq*7qTNt`CL8h& zB6U#^y_@qa9~|h@9w&A?V+hwP^J@>gDMY;R!j~E=c5!bvX6x(3#)j2FuICMEDmf35 zLBUM9uRGKKs5<5L&6>%bRg`1%7u_B#!>HI6&VPR8we$$I|9X^)KlMY++BnWB1+uYM zTv?deI%-&W)c9CxoP^rI7kZPkgeW zGd_>9$4=yes-0U)0@roN+o7)LkkS|L|JjL|O5GZ zbuAlt8j8_RqPO%5`Bh&cv+c7XwcCEOJ=d4Xcydl)O#)r~A&@oA&*b|#Q)WBBoMoT9 z1!GrxGW)qbFw9j+8*?I<;l^CSG|7(D4C7eO)!QVA?IT`}cW3h3zYE4)Pr;N3TUNr? zpE?=mXODjd&AeXZB=x{5p1QVTt{+nb9pZJ&NQmjXR*W2KC6*W@xS(BEtW699&0(Z{ zb-t|bY7cZvXpd>m9w6^E3ml4uV(V01$8-*b(vM~0Sd(3(+S#c@F+0ZJuWl=)Eo~6( z+ge~uj2V@V+6^rOk^B#RAb;O>XzOalrewHccJWAhc4P!I#QrGQW-0OfP%F$D#VE4x z2O;SicUV$9w?IuxL@@9D%JHG9YdHR7m?Jn7MZe zWxe0?3Tw9*o)J!=+C8V#LAta8v4o?pDW+lh0$v*21 zDe@9%(#nBYI=v@Y_X)<9rp}C@j$(Et~pQ@=U zX$4rV^GENmeVJ-UJJGq(1NBol)^}tZD87cVlOH;=n8v%J&s8bY@9Ds@P|6A&)nIk9 zE6R=?5yr0PT$D_2)je()|T|JO!x8>olKsG(T5_828SKPYG{!|B`f z5m>(M2<7F@fRZeMlCgq}+xzi7UJ*30L*hHG%LdNhY~`B0y5vxn@Jk!E3 zp@r(1*37YBG}-sEWY+UekbYehEvoIn{O?aBp8-x7KD-1por_CKf3&27_qK9vcPgy7 zY{8OwI~G3PlZTyXD&Xes9p_HuKB<RQ=Q{MyBKo)t8pR+&O)japYZ6 z?DL@Num;F~Yr^ryJR$PwC$WJ$Qq@m7Gkb9}SaewhWDGeQzdbAEbGK?(4TP+2qVS*qiwjRb_v}Ub#o=l(UAzJ>%Yvy3BplC8tj^~O9 zv-M#Re%O^cJmLD3o|bfdWG?7$oD)|z_Q&h5pHb2ot)LBZ$L9MTuzr#w%WPRh6`Qr$AU;Y`Q@y&90E+YRoFeb!B?dpu29e6&xpKfj6%yR8231@@f89*EtaD z4t#}!zBOdHd;lujb_D5;-vq--TS$A5A*5`TLh19Km{Jyin`gVB%f~J(Y=00KP3eiB zUfQDBtqKa``i8)$)1=sYi_F9oa45iG5D zve0xi0Fs@WL9OYHP4+=dbMXndD14abzP%9qau^mA>=e?BcL}Q3WBddD^ky;k+eC%+ z7qOhz3X&NwVON=es>vsyiDTn>(LzWXa8hjcy9}EABAx74FDh*}0aDmlNSN#mMuEO; z;dOuJH`SQsTW+V?#jl|}`93%~@cY};LjI-A5cuXPbe+_mMTK@o$AH`9XxIkYQ(r`> z_3zXa{1w734#I=uLNLX|1rpEHg8k(nHok{H`h~TU(VL#APJJRQ3gD?`;~v8L@7=L% zW^eS{eTZW?yas*qr?M`ihI0)XSbw|&Gdd8$a@UkmAgrL4U(&cXqlj9bheKt-Ua;-n zgOw}{V9{Us7!URkliaonfj5m&?#cNq>#t(-jE<~fwi1i_KA_s?!}ktS=P2auId z3WuQ-^W}^X$9-LjqDNx%x)3n?xe%&GG|}AioJYJg0yW?Zn#nR{(qmpP`3++!BSIh$ zKT+M`E%Yp;Ey~j}1;y9}WcK|ckW?26)>nH%(YtHZz*86e|GGt*pCnkC6~dgXvM9vP z3{&MMXohYOxzPX+ zWpbThIVlQDbkSCaA)!?VTL)S)@8%2g6)>L(Un3Z$C4Qo7bPh)ud{kMbsOme^jw5~DAWnc2lZRFQc-RwYkrtP*8O{e+_?r^7Mo)JFXc39n>ouJ>cb?r zel4k8GLss_d$gIlqqdzlL=OvRwRRIp^1fWx`&%b;`gt8Z+u9M|4eY@pKRgk9xaPO% z+XreolrLCZa=_f`E?9l}vyfP9gD;<(qduSSCr*rk47+U*on@f>^;anN-d>XL*+Y#% zfmpihKB+U;P>~JiJI#2#dgBR2&9LUWLJKx!b{91Edj&Hr9kBF-8_2!?5WGhp=9+-f z%GUQAC{Qk7j!ce5ce*f(JATZ6O(Q96)I2=C5<;eMUCQk1%4z-Wn8xL$nBDFbHC+v+ zaK{5ssJIUHbBtuxBnkUD&)WZSj$<6|YB!V``7LP-ZSc>G%HRZIv{ z#DvxLZHrHQJ7M)aTNc};0a7+sKvt(t7-~6yr;W}j(PWnksx3d2&~!ZnhMW);5mEky z$y{^J_jV4OIae^o8LYN2jT#$;j8KXgQqJ&nL>;+A&4Altjs}}^rApausKI5 zeVQQV^H;(AJ1m4MmB5;)Fs$|rV5MQV#n3Ymtg2@wX@7VtCe4(9_Vy8Rbx~(j zXI~-fZ|%^k7BMY+CON(z%o;w-gq3>kwOe|RB7Tsv?3Y&TGT&pBRt=oP@@ner|pE|du3pt+Wz%xpBnEN?I7f87kiO}#MX^$Us~(hj8)xRz-8 zS5f;RoeWF)2{iU=gp$?Y(dD4wSXyuc4&KiO3D<%zd^w0!F4#_*9o^ZQ06T1cB(RpP z3E&nWlC)c$cpiYKv;;s}K2LPZx5C^f%V^m1HZ1M1nUHyYBot}RQuTWsmGFe3VYl6w z!F4}HnpTS$$2d+^&7iPg1Zyo!hp6mJ(8AO4RF>~V|M(%GU6lwHySH;67WYwg4p2IVVK5|Xis>{@1G!_%JH7~0er0Q9tq|9LLh!KKT#@G01fAXlj>t2 zVvr20ZVo|1emFJUb;l&g73;^>6Pf)Eab4#@!O#7fY}*q-6ZAlkZmAX}1#guF*G5yo zqRuSs5-{m&CMFo3P(f51pdB4q(ju;v$vy<7CS#~@J=cN0nOxEZn%$weY$&x*wdQ2EPZMsgGH@*XxOA(mY<2l%Qc4yjVXK>84gEWgxf`ySU zOZuMc3RmY-=Af~pkZVf}JQPb1%F;0fo{p>-S+#<;1*NR;mZBTx22`Iii7f3ObNoNcP z&A}MR?sXdUR}To$OHDv~kObLjOZeizHTR#?Bt2(>pBuqUOURPC><7J} zgBV@?7#4CI-YJ2fSkQ4283N2vzUwS`*PNhd4?OXD$uiRW?o}p5gpqdGLgC=RVQ4MP z0;x}#cx_-m)PHX!N&?6FU*-wwQ4>0&e*YMN{xE}7sh_jKBKVA2&MXwTtLmOajn z_$jQQPEQx@rUbEU8VRcB&BFOz0a*I(in#CdK-Awa6Ys|iWS!nh89dj(cx@l{srxVt z?|Oxf6oj*;-Wx>O>Hesl_n!1V!?|D3o~ec}S32Gh>3aND7(4q8RV|KSi21R1 zn*EudJ7~fbMJ_x&z7Yb=?^E`47pz%oiBI-;VA-125Wc}0r5oB&V7JF0Im!7?v9m7C z+6-#VbD%EyID~m~T*YjT5c%;XS*m&Z&60PJJg1K%&pvGX-2I}&QiipCM4;(S-G&;3;j3!36h8)p}e*$ z_v&4xsN5PDdBy>+1t_o*zJou$4M4s1HYoGdgLVA^Fn-sTIUn|5&~pS<4t0c+?YYl< zg9mEAUn@3UsTCAw+;p1cyX5G03<`g+f$D!X5gmR?sf+ePw%ucz&Arr`es6UCdpwzs zraO}u-xnf(_JgQtEfl|K3)M!xq88(obo^u(^RLr!Ut5jfV#@awBTPxLCsQZOz9?G! z#S>o7ujC%~FC-P-2u)ve#6zxaQN7}!7&w6^+n=%{Q#sdPzjsEdT^KdJ-6bYlR6|_Y zHYymjMvUq8lx{bN;N>7&R)3tQW?$~b6Z3=4H(Mp9Uj)&^T%{Y>7t1FE7Qk=!#E%15}fJ+9uUZuwPoOGhlY zQ!R#k){@a#Ka5t(prz^-sRA-dWzqpL?^p(SRs^v~nN-Z)G?F4GjpQE4gW$N(lkfk$M8%zUq`!Jh zFo<4a#Qh|n{f)xjcVdcy;Yzt*4EcWRkImsbXn~hAK3~!gliWr_@7u<#q|}W? z2D%H1AIn7r*B%BcMPVF2$#Ph0Cq7?Z2J+epigOm+3)~wmdKqK&^8)z75FZ@_mgyS> zx$DnUOYA&|8ITImb#_!T+=4m9KY;4ix8!|l1g`nOxj(O=sJXsY2zM%l)rWcV{kDr# zHOQMajOqg4Mh(Tp1{;jNWyGdTG{Rj&16Ub(6aifGmF@~ozjbFCc@{}R zCkxWd^P*za5W)IS3nuq9hU_*TxUfSQ8-L6Zv-{Yyed|WBsPu2tzDQtMR?k5@HwaS> z{z01eA%b6{FLPR)Nm3srr8?MPYt~Tod^wy9?n?y2cm7m;?q|@vm?27cw}X>aUMxTU z4A(`TA3V!u2Q{smPKi)O?o~Zl{8+9#zg{4!cXKXwX}z#|ge#i8?~jJau4477z4Wft zi^)>lgl_>nYhauQs9W4fKV}s7G3And$1}8RX?IK=S_`AEw#8zT8d{j%1|zM4#OpyF znA*BQl$|;&=1+bIhRexR_BaPh-29+W6M@+i#*mY@4+ie<#+vu_XGsITh>4oLVE8&4 zuD^UplN3IvdLU)@pK@$^59!v2TA-)#FI<->BkAa~bdq!Gb#Yl#oy|RDxpuVZDX_p_ zzJdSAfoOPjj*QQ{vYIvFtTf-BWXIB>`QdpuQWU`&o(*S(Q~QzqwnoT!sWq>vb?(@&KB@75rB3qMFihmY@0(n*6=_d|5(&&qex588cg^ zKzZk*l(VP~N^8bZS)XBeQt885e&SqQUN>sI%X3UPMrJAZU^$aNQM9cY)=^wDiPl<63omlRdrxe}ccM5GXLzV3nWo^3y5K~gb>xdWNu(B-% zKKmfJ88oEs)tBH}1nMS*VEH2r`E^`L(6s~_)*a?Lyl-IGbASxj+*#UnceI#pjCs4f zAvZ9aa_wwbie@F4eY*#Te(S?#TU@6=fBa*&plXDEv8YSd?9DIsti=;i zdZuz&}MOV8yBXZs}VeHY?z|12qM>(K-ke9yr$tvrDo50_QoX`G&&fql0q=A z*oj(?^g{O&o)}+6pt(OssR_KxwPYJfFW51?|4pSdu^TJ-+@CqQ--NWi&p@3!4n~KE zvzo~kOw!}45`NvnwQ+aEQ9%+^&C`p;>JCy$W;e|iuaXV|a=#~ey>eGdnxA0nqg zq0H0Knk_AruvnQtTkU7T0)KD-SjqhXjYpvQr`z-;r4!04Y#?DpJSFUsQw5e#<`O`) zaSj2nPl0|n>7+Nu(zJRWwQTJ9&Ob4ExZXt~lEX@4rtkF=g-B0PnR8pF&^h@VN;?V9 zi|v_o(jg(!ZWwEl&m_q`B~yRZG*rooL_4|Fdo{> zJ$iTDSVh<-Na<#vjj6t@c)%v;18ukBdZ@ZwnwT&)D1~Y4I9R$XN!#Ex1%MVJJ zcWpB?zA<5GVfkXYlQSlEYyq|M8AXQw$T8C$6nH*YR1e9$4#Sh2Z6=?VM%$>y-5>sOW7Ln zg0dgqrR4l#YTEOPin=+my6SM0e11qFr#XkXt2bNN!G+hPUqy4CjS~5&JGD&a!~ zYMr_V!Uhh)(zY*2I(#!r$E!@ zI3#!;rh`p+q!^u|D^HX|aYYNIWY2^(%feXG1u2WZGmuI>)ZjK{4`qxAW65=EsB*#@ zXo*?@lJk=Udya!ucC>@bb8T_#ZIN24!r_nT;pleYFlBGL4AEolV8y3iSeP=Fk{tPa zR=fbMmy1}q{s-{m-k(oP2Q&Y+B4m6H$3MQAvg;Pz+4^et?n1$Dm>J8Q_yrUmDj`!MV7APgO=}EcX~|ch(uC`M_5-@C>dLn7w`Nh@+hI8O zZaCg>WAD0zu#BWM(3m$0#Sd~w^|(Q4x%Mno{wo|5mj()ZP6pwOZobTO!a^8yrzhKf zE(}wq&4<;GpOABNTbA2nFO~CC-TzLJjN~zn|D7q4{}oyHpDB_T|0BQd|11CeBk79@ z$t2_df2S`_N92n(@tN5m)56Z?7GCIJTDEP}M1L3m+p6prKPoj-VC4QuOc+11T!@NH26p8c4{9va1yzq2<>2+d-H zzmA{K z2b%bA_M+cE(8PbU7YF?VP5d`|aqvIT#DB9F`+NTbZTvU8vH$-8>r4Ot0~VP6|Ll_*kZpjpzaN%y%nA<5TXI!JHxx4wccTUpS8Zf*ILi-`r|(5T-QEl@BP`sb?&pbrY46YusF=uFNAYU zmBZnW@>;|h%;9jBL`6pXL~#V7mWBIq7=fQFBD@t}uPCqYvx4DXkzR|JaC-Q8kI?13 z;~l=={x6Bjeh@w4J>cIH74TMhPqO-tM1y#%9Klep#eSS|1BAL9A>P5>J|xha!>{-K zLOIeeB$_uVpvO@jAQ@0fn0MmsbGTlG|m>Cl06*WnpcP6+baJe5xmUlKnmopI5 z17n09hbO>~G3dM9-VR(08Qw)&yVruKxfs0Cwj~Fc;~?Zb8Q<;UqF=@#Qn^DJLV~w} zRR1OTdPp}+XJ4MWfFd?KVO8B}l%8@IShGa9-I)Th zUHY)4`wz5k^}+1$lUU%|z#a%Zjef}|;ayKQT09YhTONBc^4rP(I$VNvCkggYm7C|>gwATil1Z3b; zOg7H7u}0;4YFJ>dgKO6Sb{&WUiJGfyXv8aG;3Pvo{h^81U%V$dcj7QLH=ru!h&T=n zn9uIIyPcY~tAdiS1=&i4Kvg6bTHoa0vZJAN?8C3Lx77`XA4#CH(J8pZd;)0@Z=(E1 z=3`pxALL$e4{Otq17k0`vktxt8r^!x1WIK<;T|Wrn$}A#9&mBxrqYTt1@*L$e-p{| z`-7;xoCogLT+vA&97L^BK{8SSrOq1@wz!roayZBJH=B&FZ_bA2c2ZC#Wsa}H!L3c73?0MnvL5Q*gf=wNy8Op< zcSjj%oid;H=^SI4t$Ueu$uT&x%@SQ=%y1Uhm*#&Cftz_*p!38H1*#&WJ=sc)pX*|P7$uvE zg>avD1=;AZkVtxPG1p-~vm_`BRW@uUvo{_kL$=%_ZW^v2w&DbBbKuf0%7{_VeL;2MOFFIuv$2D`DbKwK5THdq_`%B^1{e6R>m#j&M2%YxCAi7elp3I(?Qg z3~YCNq@{+1v^qu}FSKibYLNt90}V`)I7ThRc9X@5YGmUIB|MO9PvWn+;|kLvO8mBy znTnbuE@L%msdmM{Z8mWKxd$H3AB|%))ZxJAV0e}50_9_Ok|je|QGt_^aCIc$Yw7o- zsM#KM?zFN(r_&*2oGWASU;=DxKFWlsdBrsW4t%4`^7om*yTnl!? zmo0g;m;X6w+E+{~%qrM<@28O^lU5OD{wt(kZ8WjU98XS8jv}7}!og#uA$%K^35`kF zIO}L89`+uA8s1|eX!8d~Np%E@E=z)w+A`31wugos8pX;>ye8WPhT@xYf1I-ACEahZ zo%rg=!?~1vMk3}mEgSQNjLZo{%gpoU?G#nK>}}ZWTEjJ|9a?Pk|>pDjC}Z9Zb5+ z4{<9Gk(gup(5NkoY#*ySO9j0}MbYy_HY}*RO3Mo+@gDz9rgrmN!VFQwi#>L@S*e+tn!aEo zAKSsCyTN!^rHAau=5QYpH`=%75bfBuo~~VZh*@`91G1J=x?=|~XCHY+?cLmI-#aNV zvXDZjkSis6-GvW`WY-JQ z>swFLJ9m?`ZZ}$^U=6NfshD^p2@Eb=Ae{$ZlH_|*P#m6(a;C571fNzK{wfx2w`HTO zK_-^034@^xKHyw3oG8jULO`tz@d;#Hmjre9L1!-q;zvo!-Jpd_Qem6!K+ zour?Zrr;ebFYu0dM!$WWhMH$qFd>ne_@XKU0{1DRyUh%u+N{ic9&HEtGfy)X+-GFQ zo-~k|beOhJZ>NLQ(m=wumc3~<9b06%Ad^%`0tQtuvkK)vy&xT|r*q-;-54+_O$4(s zqfuJWnCR@x!I3Ghz_rSP;a%ehL8Fonr z(cmwgbX8Y2*cCrD4d%Y2?FvUo!qC|;rEUpE6jl(yNuyA%SQEzxNHO-Ew)D7oKAC@E zD2xCTe0Y`*Z(K-*c%{$e!U8)`w2$OAt;>W@hH>~JY$1L)l?L-Vlc<`DK53KnV2KDIkO66!G5GwWLq*4SnR&Nc0#XoOdA} ziw=c@fKDXMu8YSKxmZlN^rUi2(FpXK;SaZ73Zv_nRA4v{h)}USMl=|aGl#XX zos>w1swERuKYLgZ@{Gzzoi+_KU|_a^Gw3{La9+td+L9PSQVqxAmNguzu~r1z%9f&m z$oa_!T++a$gP*y6IuaL4ts#?>J<)dZG&rO_2evN0&wLTqf;Y_BsyXKCnaRb5a4B37 zM2*ey$zeZCn(zlXUyz75PrV|8RIAATxL$#me5`yN%ut0#T^O$(7C>dt*|r3i4O_6r#%%~yb`dPWB~s}BTxxQgK9Y; zR5|Jean-q`HoK8le=Mg0GTs;~wwI}m0j7PJ6zb|*quH4PV*OwYl$TFMX_@=9nVA3& zzNuqdfEG=b4<>FUFX+X$wkW*BvMTWU2dXT1oLJpEM6Q|ovPXS3lDh3>Oy1{M>g9HW zz6#$)*GZbAO!lo2h>5N*Lu(SS4}gB1!6w!c+%-h+R3BZP#?9Z*uKW zS$R3^UALVncrY6kdVFxWUKXo&+nzkK6re0v;F&x<5N{43q1s8H`&I|_4ILpTY$sKb zosTy?0PdYng}m$2;C77$zDZ0&2lfMTJp^R_rwoS7vW2FS<4hlaEm_<>54&wydU>ll zR&Q@)=%5(fl2A?)1^uDFbvRo+ql1Kams7#l9aLoR22xb32u_CAs0j0tu459oK8C?K zYnB}B-Xnni8fWRqs*$N5HRdBSL5AwVz+-OZklOdbKr>1%16FCX=cHX`q?%;7HP3jp2jWu!}32W%48GSxE**tlT= zxa(UCxK!PsL~9ye9TEl8*Q_GLsvN5-WvXeaXEs(n;)8o3_84~e1f45qO&1++qc)v4 ziT+3_=voCGtE%7F9tLU`H04`U@%#mh!XaX8R=?O zB-?Cg*Riqi`Dzkm$)rJY*A=Rt=85&qk4aL2269GNlW|Ke@tXA&`m)Cfldi0U(;x28 zh0849{<9pK_1X~mFLcxRMX#vQ_+gmnx)R23N+uW7of#!#52(?6%}UyFh&KX=#P-t~ z=R_!dIUo0*JWk&%?V_KY?lVr25!fD)3Lc>q)b_Lqo1$lnbCg<`{q|PKHWrZs;-9E< zK?bbfphd=5UFFJkts_ag-r(>h8K33FaWzC^v3VFz-@-Tun3#f*pC6H^r4va)Oc0j) zKVjahPlE;B8K^Bj64#ne!`MsPX?fpV(7Bn7lg8TPiP}HNkijyiLF$oV>~;mI04#Lo1@Vf z4N}k&3yUtM!^Y^1q+wGw7-sCIy5iTUM;|}Fk#1*NuPv+6j-QDij!&SswACRfAO@!% zbA<%r#k}ubO=92Xz=jj!$WmrGwq5EZ(^}ms?Z0qP!FxDK$V)+s(LE$QJ%P;I-Aq(l zU3l@TfNDa%co)kVX0hM*{JKzY5{3o(kf$1N*{AxOxGh*uMP_Hhhir4;U+V%DP(%G9 z0s5`+_t5x7$K0)1G#iBSrpw7bU~ z`;|A+#-2-6dge~h%8|ha@dW&$_MGOXbkf4KVbtxiI!)n^!QJDFNaSQ)c4C$Zv`k-1 zu6@)3^>T6K_YH#ct6%coxkKQ{&?vYhU_meX>Oxas3K`=&5ksBUkc+};z%QAClAWh$ ziAXf<4i8}q)^u`5MQ)@zho91>M=nsgFa<%`gHMl6gyriebdBO%$f ziahL?jxvM!P*S0U%QsDk1g`3)fo}8AD`zX?u96N{Kg>e4@;UUy6L&1FosY(U6{duun?yeS7;A}ATSRd0`3~wT zZAyCgs=$32Rg$BAn2y?MkB-aD>HNeCB=?vvF}dW1$FFcSwcHaeyN{;cKTmZ5(i}#1MGY zPiS&*7znSlVQS)D(9Nrr$)J7viOM7fuN-y*YgPd0ghAxJ>toWH8V@-wF;uNL3D4N3 z(zPnJH2i)R9?d;LWYr##dIM9CU8IcrWw+3{f(5Ws*aWLYoM1td5Lnc|AR<<`s<`1* z4tPv>};c=?kPB)iNnBe=SV$0!77z+A-nj_(~+=&F5GvAzS7~% zD_mz1WZY3d$Oh{|&y&Jciy(U40#qXcP{E7$pu}&CVwEj;E1AICT|?lYY!+eVJK zSYX`ncmA!`n=#I4%rR%bqZjtS&dSV;XdCiAJ-8P!uS5RJCXn7jyE` ziAtg~nG*BdRIXD25AJouJ(!7g8&%+S(;X7Zn!>Y}>Y(S*#XdQifl_a*a94LTu^-Y# zWn6}%O^zA0%m_oy<-@d|-x-!0DnW!05|2Iep#8KPTV10{?t>ZXIR@cPQ)lwN)R8r+ zT!>TKV!i8M>>EIml`cw|iym5)C;rm3z!ZRRtb35tm+C&O3 ziP3AHZ_p3Jx%6U4AbRIT!&lB(DqlVoXkaIC9ur11KDbghUOtcsyG6@vYRH!xQt&B& zAycY{?~#=5lVxhef@8-tf7-=#A?isQ7ivr)0d40|Wp zk~4Xo;d9`+&J5i6HGubxPZ(=zOt9^S9Ldq02!)5e z*?m(}*d31(AaQIw`n_Gnidu-n8?pkXPS=IAQvR5ot%lweTqr0LguS;y;q>x_Wcw;b zuv3qKmD7u>hE3C_PvW(pZf7h}8C^-u$!Ju)pTn9+Xrr`{4u-#S0#oCIRSm0Gv*X6? zVi^5p@I=9a^jsVa723WinYx-(b{rrFRfjTTHS@@dKh9F2OKr@UZcTj60)Em}qW4Q@ zg0yEYT{w+FQ`MEYDyWu)0asV5sQVpa+%*V1I5ebFP^KF?Xx0 zwh6E_E=Cd$EJ%ivlgj84&50;i8&4FR#i&(@5$td?#(_$R-G8e9__O--s|4`#yZ^=S z^?<+2&u^(_>2o~)Kd)sk_)*Jt{cpMQ&uSKr-S@$(X8kL#78~SB7vy;Z=ybEDmPU{t z_lcO#*Ta2f1R5H5qh9_l*r?==*QGf?N=wkuU=`ZSslh?7ep1gz(5g;{_(WwudhlUL zkxU|9%I(BAAq{9a1Ib^cF^*TK4xgG2^%KmXV_;jDGwX?v(^E8QoK@k*N z2%?^MF}#|65C*bD7EjUtW{vt6v_G?i;7_D^Ma$n%FCJhn==rmwX$i-N*HOZ2o!Y^h z{r7Dm9=uf^(f?f+#h)7{ezau$sXyvhZ=e z|DA}FeuIdA5k-UFAmU#{(eO8j_!m($`VAufMHG#HgNT0-MU&qk;$K8@@^29Fuc64{ F{15PjC!qiU literal 0 HcmV?d00001 diff --git a/src/neural_networks/NN_test.jl b/src/neural_networks/NN_test.jl index a5dae11..68306ba 100644 --- a/src/neural_networks/NN_test.jl +++ b/src/neural_networks/NN_test.jl @@ -3,7 +3,13 @@ using Distributed addprocs(2) @everywhere include("bound_tightening_new.jl") +@everywhere ENV = [Gurobi.Env() for i in 1:nprocs()]; + +include("bound_tightening_new.jl") +ENV = [Gurobi.Env() for i in 1:nprocs()]; BSON.@load string(@__DIR__)*"/NN_paraboloid.bson" model +BSON.@load string(@__DIR__)*"/NN_medium.bson" model +BSON.@load string(@__DIR__)*"/NN_large.bson" model init_ub = [1.0, 1.0] init_lb = [-1.0, -1.0] @@ -12,18 +18,7 @@ data = rand(Float32, (2, 1000)) .- 0.5f0; x_train = data[:, 1:750]; y_train = [sum(x_train[:, col].^2) for col in 1:750]; -model = Chain( - Dense(2 => 30, relu), - Dense(30 => 50, relu), - Dense(50 => 1, relu) -) - -@everywhere ENV = [Gurobi.Env() for i in 1:nprocs()]; - -ENV = [Gurobi.Env() for i in 1:nprocs()]; -include("bound_tightening_new.jl") - -@time jump_model, bounds_x, bounds_s = NN_to_MIP(model, init_ub, init_lb; tighten_bounds=true, distributed=true); +@time jump_model, bounds_x, bounds_s = NN_to_MIP(model, init_ub, init_lb; tighten_bounds=true, distributed=false); jump_model bounds_x bounds_s diff --git a/src/neural_networks/NN_test.txt b/src/neural_networks/NN_test.txt new file mode 100644 index 0000000..216811c --- /dev/null +++ b/src/neural_networks/NN_test.txt @@ -0,0 +1,56 @@ +NN_paraboloid, bound tightening, not distributed +0.020664 seconds (14.06 k allocations: 597.891 KiB) +0.020988 seconds (14.06 k allocations: 597.891 KiB) +0.020137 seconds (14.06 k allocations: 597.891 KiB) +0.021308 seconds (14.06 k allocations: 597.891 KiB) + +Forward pass for 750 elements +0.229151 seconds (451.07 k allocations: 15.874 MiB, 9.32% compilation time) +0.238844 seconds (451.07 k allocations: 15.874 MiB, 4.66% gc time, 10.04% compilation time) +0.225986 seconds (451.07 k allocations: 15.937 MiB, 9.49% compilation time) + +NN_paraboloid, no bound tightening +0.001234 seconds (5.25 k allocations: 235.828 KiB) +0.001441 seconds (5.29 k allocations: 238.625 KiB) +0.001283 seconds (5.28 k allocations: 238.484 KiB) + +Forward pass +0.210862 seconds (424.18 k allocations: 15.053 MiB, 10.88% compilation time) +0.217571 seconds (424.18 k allocations: 15.053 MiB, 10.60% compilation time) +0.216805 seconds (424.18 k allocations: 15.085 MiB, 4.93% gc time, 9.79% compilation time) + +NN_medium, bound tightening, not distributed +1.754903 seconds (137.68 k allocations: 6.446 MiB) +1.768755 seconds (137.68 k allocations: 6.446 MiB) +1.753357 seconds (137.64 k allocations: 6.443 MiB) + +Forward pass +1.342592 seconds (705.18 k allocations: 25.302 MiB, 0.74% gc time, 1.55% compilation time) +1.331219 seconds (705.13 k allocations: 25.298 MiB, 0.64% gc time, 1.67% compilation time) +1.362238 seconds (705.13 k allocations: 25.298 MiB, 1.73% compilation time) + +NN_medium, no bound tightening +0.002844 seconds (26.63 k allocations: 1.667 MiB) +0.006448 seconds (26.59 k allocations: 1.664 MiB) +0.002951 seconds (26.59 k allocations: 1.664 MiB) + +Forward pass +1.700944 seconds (632.39 k allocations: 23.078 MiB, 1.39% compilation time) +1.731487 seconds (632.39 k allocations: 23.078 MiB, 0.49% gc time, 1.27% compilation time) +1.753971 seconds (632.39 k allocations: 23.080 MiB, 1.23% compilation time) + +NN_large, no bound tightening +0.005663 seconds (69.76 k allocations: 5.048 MiB) +0.006282 seconds (69.82 k allocations: 5.055 MiB) +0.006192 seconds (69.76 k allocations: 5.048 MiB) + +Forward pass +17.214665 seconds (820.12 k allocations: 31.096 MiB, 0.07% gc time, 0.13% compilation time) +17.237849 seconds (820.11 k allocations: 31.096 MiB, 0.06% gc time, 0.13% compilation time) + +NN_large, bound tightening with 0.5 sec time limit +55.179892 seconds (1.87 M allocations: 107.170 MiB, 0.06% gc time, 0.81% compilation time) + +Forward pass +15.014602 seconds (1.07 M allocations: 38.809 MiB, 0.07% gc time, 0.15% compilation time) +15.002895 seconds (1.07 M allocations: 38.807 MiB, 0.06% gc time, 0.15% compilation time) diff --git a/src/neural_networks/bound_tightening_new.jl b/src/neural_networks/bound_tightening_new.jl index 4648861..569bb37 100644 --- a/src/neural_networks/bound_tightening_new.jl +++ b/src/neural_networks/bound_tightening_new.jl @@ -21,6 +21,7 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect jump_model = Model() set_optimizer(jump_model, () -> Gurobi.Optimizer(ENV[myid()])) set_silent(jump_model) + set_attribute(jump_model, "TimeLimit", 0.1) @variable(jump_model, x[layer = 0:K, neurons(layer)]) @variable(jump_model, s[layer = 0:K, neurons(layer)]) @@ -39,6 +40,8 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect # TODO: For parallelization the model must be copied for each neuron in a new layer to prevent data races + println("\nLAYER $layer") + if tighten_bounds if distributed @sync @distributed for neuron in 1:neuron_count[layer] @@ -46,6 +49,7 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect end else for neuron in 1:neuron_count[layer] + print("$neuron ") ub_x[neuron], ub_s[neuron] = calculate_bounds(jump_model, layer, neuron, W, b, neurons) end end @@ -91,11 +95,13 @@ function calculate_bounds(model::JuMP.Model, layer, neuron, W, b, neurons) @objective(model, Max, x[layer, neuron]) optimize!(model) - ub_x = objective_value(model) + @assert primal_status(model) == MOI.FEASIBLE_POINT "No solution found in time limit." + ub_x = objective_bound(model) @objective(model, Max, s[layer, neuron]) optimize!(model) - ub_s = objective_value(model) + @assert primal_status(model) == MOI.FEASIBLE_POINT "No solution found in time limit." + ub_s = objective_bound(model) delete(model, x_con) delete(model, s_con) @@ -133,11 +139,13 @@ function calculate_bounds_copy(input_model::JuMP.Model, layer, neuron, W, b, neu @objective(model, Max, x[layer, neuron]) optimize!(model) - ub_x = objective_value(model) + @assert primal_status(model) == MOI.FEASIBLE_POINT "No solution found in time limit." + ub_x = objective_bound(model) @objective(model, Max, s[layer, neuron]) optimize!(model) - ub_s = objective_value(model) + @assert primal_status(model) == MOI.FEASIBLE_POINT "No solution found in time limit." + ub_s = objective_bound(model) return ub_x, ub_s end From 417747fb5b2e78dc05f9589c82d977845aa8d01a Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Thu, 25 Jan 2024 16:41:53 +0200 Subject: [PATCH 08/32] added new NN formulation --- src/neural_networks/NN_test_serra.jl | 27 +++++ src/neural_networks/NN_test_serra_parallel.jl | 31 ++++++ src/neural_networks/bound_tightening_serra.jl | 90 ++++++++++++++++ .../bound_tightening_serra_parallel.jl | 100 ++++++++++++++++++ 4 files changed, 248 insertions(+) create mode 100644 src/neural_networks/NN_test_serra.jl create mode 100644 src/neural_networks/NN_test_serra_parallel.jl create mode 100644 src/neural_networks/bound_tightening_serra.jl create mode 100644 src/neural_networks/bound_tightening_serra_parallel.jl diff --git a/src/neural_networks/NN_test_serra.jl b/src/neural_networks/NN_test_serra.jl new file mode 100644 index 0000000..80f959c --- /dev/null +++ b/src/neural_networks/NN_test_serra.jl @@ -0,0 +1,27 @@ +include("bound_tightening_serra.jl") +ENV = Gurobi.Env(); + +BSON.@load string(@__DIR__)*"/NN_paraboloid.bson" model +BSON.@load string(@__DIR__)*"/NN_medium.bson" model +BSON.@load string(@__DIR__)*"/NN_large.bson" model + +init_ub = [1.0, 1.0] +init_lb = [-1.0, -1.0] + +data = rand(Float32, (2, 1000)) .- 0.5f0; +x_train = data[:, 1:750]; +y_train = [sum(x_train[:, col].^2) for col in 1:750]; + +@time jump_model, bounds_x, bounds_s = NN_to_MIP(model, init_ub, init_lb; tighten_bounds=true); +jump_model +bounds_x +bounds_s +@time [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750]; +vec(model(x_train)) ≈ [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750] + +@time jump_model, bounds_x, bounds_s = NN_to_MIP(model, init_ub, init_lb; tighten_bounds=false); +jump_model +bounds_x +bounds_s +@time [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750]; +vec(model(x_train)) ≈ [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750] \ No newline at end of file diff --git a/src/neural_networks/NN_test_serra_parallel.jl b/src/neural_networks/NN_test_serra_parallel.jl new file mode 100644 index 0000000..e5c8bbc --- /dev/null +++ b/src/neural_networks/NN_test_serra_parallel.jl @@ -0,0 +1,31 @@ +using Distributed + +addprocs(4) + +@everywhere include("bound_tightening_serra_parallel.jl") +@everywhere ENV = [Gurobi.Env() for i in 1:nprocs()]; + +BSON.@load string(@__DIR__)*"/NN_paraboloid.bson" model +BSON.@load string(@__DIR__)*"/NN_medium.bson" model +BSON.@load string(@__DIR__)*"/NN_large.bson" model + +init_ub = [1.0, 1.0] +init_lb = [-1.0, -1.0] + +data = rand(Float32, (2, 1000)) .- 0.5f0; +x_train = data[:, 1:750]; +y_train = [sum(x_train[:, col].^2) for col in 1:750]; + +@time jump_model, bounds_x, bounds_s = NN_to_MIP(model, init_ub, init_lb; tighten_bounds=true); +jump_model +bounds_x +bounds_s +@time [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750]; +vec(model(x_train)) ≈ [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750] + +@time jump_model, bounds_x, bounds_s = NN_to_MIP(model, init_ub, init_lb; tighten_bounds=false); +jump_model +bounds_x +bounds_s +@time [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750]; +vec(model(x_train)) ≈ [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750] \ No newline at end of file diff --git a/src/neural_networks/bound_tightening_serra.jl b/src/neural_networks/bound_tightening_serra.jl new file mode 100644 index 0000000..451fdce --- /dev/null +++ b/src/neural_networks/bound_tightening_serra.jl @@ -0,0 +1,90 @@ +using BSON +using Flux +using JuMP +using Gurobi + +function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}; tighten_bounds=false) + + K = length(NN_model) # number of layers (input layer not included) + W = [Flux.params(NN_model)[2*k-1] for k in 1:K] + b = [Flux.params(NN_model)[2*k] for k in 1:K] + + input_length = Int((length(W[1]) / length(b[1]))) + neuron_count = [length(b[k]) for k in eachindex(b)] + neurons(layer) = layer == 0 ? [i for i in 1:input_length] : [i for i in 1:neuron_count[layer]] + + @assert input_length == length(init_ub) == length(init_lb) "Initial bounds arrays must be the same length as the input layer" + + # build model up to second layer + jump_model = Model() + set_optimizer(jump_model, () -> Gurobi.Optimizer(ENV)) + set_silent(jump_model) + set_attribute(jump_model, "TimeLimit", 0.25) + + @variable(jump_model, x[layer = 0:K, neurons(layer)]) + @variable(jump_model, s[layer = 0:K, neurons(layer)]) + @variable(jump_model, z[layer = 0:K, neurons(layer)]) + + @constraint(jump_model, [j = 1:input_length], x[0, j] <= init_ub[j]) + @constraint(jump_model, [j = 1:input_length], x[0, j] >= init_lb[j]) + + bounds_x = Vector{Vector}(undef, K) + bounds_s = Vector{Vector}(undef, K) + + for layer in 1:K # hidden layers to output layer - second layer and up + + ub_x = fill(1000.0, length(neurons(layer))) + ub_s = fill(1000.0, length(neurons(layer))) + + if tighten_bounds + for neuron in 1:neuron_count[layer] + ub_x[neuron], ub_s[neuron] = calculate_bounds(jump_model, layer, neuron, W, b, neurons) + end + end + + for neuron in 1:neuron_count[layer] + + @constraint(jump_model, x[layer, neuron] >= 0) + @constraint(jump_model, s[layer, neuron] >= 0) + set_binary(z[layer, neuron]) + + @constraint(jump_model, x[layer, neuron] <= ub_x[neuron] * (1 - z[layer, neuron])) + @constraint(jump_model, s[layer, neuron] <= ub_s[neuron] * z[layer, neuron]) + + @constraint(jump_model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1, i] for i in neurons(layer-1))) + + end + + bounds_x[layer] = ub_x + bounds_s[layer] = ub_s + end + + return jump_model, bounds_x, bounds_s +end + +function calculate_bounds(model::JuMP.Model, layer, neuron, W, b, neurons) + + @objective(model, Max, b[layer][neuron] + sum(W[layer][neuron, i] * model[:x][layer-1, i] for i in neurons(layer-1))) + optimize!(model) + @assert primal_status(model) == MOI.FEASIBLE_POINT "No solution found in time limit." + + ub_x = max(objective_bound(model), 0.0) + + set_objective_sense(model, MIN_SENSE) + optimize!(model) + @assert primal_status(model) == MOI.FEASIBLE_POINT "No solution found in time limit." + ub_s = max(-objective_bound(model), 0.0) + + return ub_x, ub_s +end + +function forward_pass!(jump_model::JuMP.Model, input::Vector{Float32}) + @assert length(input) == length(jump_model[:x][0, :]) "Incorrect input length." + + [fix(jump_model[:x][0, i], input[i], force=true) for i in eachindex(input)] + optimize!(jump_model) + + (last_layer, outputs) = maximum(keys(jump_model[:x].data)) + result = value.(jump_model[:x][last_layer, :]) + return [result[i] for i in 1:outputs] +end \ No newline at end of file diff --git a/src/neural_networks/bound_tightening_serra_parallel.jl b/src/neural_networks/bound_tightening_serra_parallel.jl new file mode 100644 index 0000000..b73fac9 --- /dev/null +++ b/src/neural_networks/bound_tightening_serra_parallel.jl @@ -0,0 +1,100 @@ +using BSON +using Flux +using JuMP +using Gurobi +using Distributed +using SharedArrays + +function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}; tighten_bounds=false) + + K = length(NN_model) # number of layers (input layer not included) + W = [Flux.params(NN_model)[2*k-1] for k in 1:K] + b = [Flux.params(NN_model)[2*k] for k in 1:K] + + input_length = Int((length(W[1]) / length(b[1]))) + neuron_count = [length(b[k]) for k in eachindex(b)] + neurons(layer) = layer == 0 ? [i for i in 1:input_length] : [i for i in 1:neuron_count[layer]] + + @assert input_length == length(init_ub) == length(init_lb) "Initial bounds arrays must be the same length as the input layer" + + # build model up to second layer + jump_model = Model() + set_optimizer(jump_model, () -> Gurobi.Optimizer(ENV[myid()])) + set_silent(jump_model) + + @variable(jump_model, x[layer = 0:K, neurons(layer)]) + @variable(jump_model, s[layer = 0:K, neurons(layer)]) + @variable(jump_model, z[layer = 0:K, neurons(layer)]) + + @constraint(jump_model, [j = 1:input_length], x[0, j] <= init_ub[j]) + @constraint(jump_model, [j = 1:input_length], x[0, j] >= init_lb[j]) + + bounds_x = Vector{Vector}(undef, K) + bounds_s = Vector{Vector}(undef, K) + + for layer in 1:K # hidden layers to output layer - second layer and up + + ub_x = fill(1000.0, length(neurons(layer))) |> SharedArray + ub_s = fill(1000.0, length(neurons(layer))) |> SharedArray + + if tighten_bounds + @sync @distributed for neuron in 1:neuron_count[layer] + model = copy_model(jump_model) + ub_x[neuron], ub_s[neuron] = calculate_bounds(model, layer, neuron, W, b, neurons) + end + end + + for neuron in 1:neuron_count[layer] + + @constraint(jump_model, x[layer, neuron] >= 0) + @constraint(jump_model, s[layer, neuron] >= 0) + set_binary(z[layer, neuron]) + + @constraint(jump_model, x[layer, neuron] <= ub_x[neuron] * (1 - z[layer, neuron])) + @constraint(jump_model, s[layer, neuron] <= ub_s[neuron] * z[layer, neuron]) + + @constraint(jump_model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1, i] for i in neurons(layer-1))) + + end + + bounds_x[layer] = ub_x + bounds_s[layer] = ub_s + end + + return jump_model, bounds_x, bounds_s +end + +function copy_model(input_model) + model = copy(input_model) + set_optimizer(model, () -> Gurobi.Optimizer(ENV[myid()])) + set_silent(model) + set_attribute(model, "TimeLimit", 0.25) + return model +end + +function calculate_bounds(model::JuMP.Model, layer, neuron, W, b, neurons) + + @objective(model, Max, b[layer][neuron] + sum(W[layer][neuron, i] * model[:x][layer-1, i] for i in neurons(layer-1))) + optimize!(model) + @assert primal_status(model) == MOI.FEASIBLE_POINT "No solution found in time limit." + + ub_x = max(objective_bound(model), 0.0) + + set_objective_sense(model, MIN_SENSE) + optimize!(model) + @assert primal_status(model) == MOI.FEASIBLE_POINT "No solution found in time limit." + ub_s = max(-objective_bound(model), 0.0) + + return ub_x, ub_s +end + +function forward_pass!(jump_model::JuMP.Model, input::Vector{Float32}) + @assert length(input) == length(jump_model[:x][0, :]) "Incorrect input length." + + [fix(jump_model[:x][0, i], input[i], force=true) for i in eachindex(input)] + optimize!(jump_model) + + (last_layer, outputs) = maximum(keys(jump_model[:x].data)) + result = value.(jump_model[:x][last_layer, :]) + return [result[i] for i in 1:outputs] +end \ No newline at end of file From ca5a824956b4b0890937b09ffe8d06fe26f14e4a Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Fri, 26 Jan 2024 16:04:35 +0200 Subject: [PATCH 09/32] changed last layer to use identity activation --- src/neural_networks/NN_test_serra.jl | 10 +++++++--- src/neural_networks/bound_tightening_serra.jl | 20 +++++++++++++++---- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/neural_networks/NN_test_serra.jl b/src/neural_networks/NN_test_serra.jl index 80f959c..ff8f820 100644 --- a/src/neural_networks/NN_test_serra.jl +++ b/src/neural_networks/NN_test_serra.jl @@ -1,9 +1,13 @@ include("bound_tightening_serra.jl") ENV = Gurobi.Env(); -BSON.@load string(@__DIR__)*"/NN_paraboloid.bson" model -BSON.@load string(@__DIR__)*"/NN_medium.bson" model -BSON.@load string(@__DIR__)*"/NN_large.bson" model +model = Chain( + Dense(2 => 10, relu), + Dense(10 => 30, relu), + Dense(30 => 30, relu), + Dense(30 => 10, relu), + Dense(10 => 1) +) init_ub = [1.0, 1.0] init_lb = [-1.0, -1.0] diff --git a/src/neural_networks/bound_tightening_serra.jl b/src/neural_networks/bound_tightening_serra.jl index 451fdce..1eb9c2e 100644 --- a/src/neural_networks/bound_tightening_serra.jl +++ b/src/neural_networks/bound_tightening_serra.jl @@ -6,6 +6,9 @@ using Gurobi function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}; tighten_bounds=false) K = length(NN_model) # number of layers (input layer not included) + @assert reduce(&, [NN_model[i].σ == relu for i in 1:K-1]) "Neural network must use the relu activation function." + @assert NN_model[K].σ == identity "Neural network must use the identity function for the output layer." + W = [Flux.params(NN_model)[2*k-1] for k in 1:K] b = [Flux.params(NN_model)[2*k] for k in 1:K] @@ -19,11 +22,11 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect jump_model = Model() set_optimizer(jump_model, () -> Gurobi.Optimizer(ENV)) set_silent(jump_model) - set_attribute(jump_model, "TimeLimit", 0.25) + #set_attribute(jump_model, "TimeLimit", 0.25) @variable(jump_model, x[layer = 0:K, neurons(layer)]) - @variable(jump_model, s[layer = 0:K, neurons(layer)]) - @variable(jump_model, z[layer = 0:K, neurons(layer)]) + @variable(jump_model, s[layer = 0:K-1, neurons(layer)]) + @variable(jump_model, z[layer = 0:K-1, neurons(layer)]) @constraint(jump_model, [j = 1:input_length], x[0, j] <= init_ub[j]) @constraint(jump_model, [j = 1:input_length], x[0, j] >= init_lb[j]) @@ -31,13 +34,16 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect bounds_x = Vector{Vector}(undef, K) bounds_s = Vector{Vector}(undef, K) - for layer in 1:K # hidden layers to output layer - second layer and up + for layer in 1:K-1 # hidden layers ub_x = fill(1000.0, length(neurons(layer))) ub_s = fill(1000.0, length(neurons(layer))) + println("\nLAYER $layer") + if tighten_bounds for neuron in 1:neuron_count[layer] + print("$neuron ") ub_x[neuron], ub_s[neuron] = calculate_bounds(jump_model, layer, neuron, W, b, neurons) end end @@ -59,6 +65,12 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect bounds_s[layer] = ub_s end + # output layer + bounds = [calculate_bounds(jump_model, K, neuron, W, b, neurons) for neuron in neurons(K)] + bounds_x[K] = map(pair -> pair[1], bounds) + bounds_s[K] = map(pair -> pair[2], bounds) + @constraint(jump_model, [neuron in 1:neuron_count[K]], x[K, neuron] == b[K][neuron] + sum(W[K][neuron, i] * x[K-1, i] for i in neurons(K-1))) + return jump_model, bounds_x, bounds_s end From 2821774bc259fb16554cc6ce9f9af4e47ac388ce Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Mon, 29 Jan 2024 14:43:44 +0200 Subject: [PATCH 10/32] refactored multiprocessing bound tightening --- src/neural_networks/NN_test_serra.jl | 41 ++++++--- src/neural_networks/bound_tightening_serra.jl | 42 ++------- .../bound_tightening_serra_parallel.jl | 88 +++---------------- 3 files changed, 50 insertions(+), 121 deletions(-) diff --git a/src/neural_networks/NN_test_serra.jl b/src/neural_networks/NN_test_serra.jl index ff8f820..10c4e59 100644 --- a/src/neural_networks/NN_test_serra.jl +++ b/src/neural_networks/NN_test_serra.jl @@ -1,14 +1,3 @@ -include("bound_tightening_serra.jl") -ENV = Gurobi.Env(); - -model = Chain( - Dense(2 => 10, relu), - Dense(10 => 30, relu), - Dense(30 => 30, relu), - Dense(30 => 10, relu), - Dense(10 => 1) -) - init_ub = [1.0, 1.0] init_lb = [-1.0, -1.0] @@ -16,6 +5,27 @@ data = rand(Float32, (2, 1000)) .- 0.5f0; x_train = data[:, 1:750]; y_train = [sum(x_train[:, col].^2) for col in 1:750]; +using Distributed + +include("bound_tightening_serra.jl") +include("bound_tightening_serra_parallel.jl") + +addprocs(7) +@everywhere include("bound_tightening_serra_parallel.jl") +@everywhere gurobi_env = [Gurobi.Env() for i in 1:8]; +gurobi_env = [Gurobi.Env() for i in 1:8]; + +using Random +Random.seed!(1234); + +model = Chain( + Dense(2 => 10, relu), + Dense(10 => 30, relu), + Dense(30 => 20, relu), + Dense(20 => 5, relu), + Dense(5 => 1) +); + @time jump_model, bounds_x, bounds_s = NN_to_MIP(model, init_ub, init_lb; tighten_bounds=true); jump_model bounds_x @@ -23,6 +33,15 @@ bounds_s @time [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750]; vec(model(x_train)) ≈ [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750] +include("bound_tightening.jl") + +@time U, L = bound_tightening(model, [i<=2 ? 1.0 : 1000.0 for i in 1:2+10+30+20+5+1], [i<=2 ? -1.0 : -1000.0 for i in 1:2+10+30+20+5+1]) + +using Plots +plot(collect(Iterators.flatten(bounds_x))) +plot!(collect(Iterators.flatten(bounds_s))) +plot(collect(Iterators.flatten(bounds_x)) .- U[3:end-1]) + @time jump_model, bounds_x, bounds_s = NN_to_MIP(model, init_ub, init_lb; tighten_bounds=false); jump_model bounds_x diff --git a/src/neural_networks/bound_tightening_serra.jl b/src/neural_networks/bound_tightening_serra.jl index 1eb9c2e..1914176 100644 --- a/src/neural_networks/bound_tightening_serra.jl +++ b/src/neural_networks/bound_tightening_serra.jl @@ -1,9 +1,8 @@ -using BSON using Flux using JuMP -using Gurobi +using Distributed -function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}; tighten_bounds=false) +function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}; tighten_bounds=false, big_M=1000.0) K = length(NN_model) # number of layers (input layer not included) @assert reduce(&, [NN_model[i].σ == relu for i in 1:K-1]) "Neural network must use the relu activation function." @@ -20,9 +19,7 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect # build model up to second layer jump_model = Model() - set_optimizer(jump_model, () -> Gurobi.Optimizer(ENV)) - set_silent(jump_model) - #set_attribute(jump_model, "TimeLimit", 0.25) + set_solver_params!(jump_model) @variable(jump_model, x[layer = 0:K, neurons(layer)]) @variable(jump_model, s[layer = 0:K-1, neurons(layer)]) @@ -31,21 +28,19 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect @constraint(jump_model, [j = 1:input_length], x[0, j] <= init_ub[j]) @constraint(jump_model, [j = 1:input_length], x[0, j] >= init_lb[j]) - bounds_x = Vector{Vector}(undef, K) - bounds_s = Vector{Vector}(undef, K) + bounds_x = Vector{Vector}(undef, K-1) + bounds_s = Vector{Vector}(undef, K-1) for layer in 1:K-1 # hidden layers - ub_x = fill(1000.0, length(neurons(layer))) - ub_s = fill(1000.0, length(neurons(layer))) + ub_x = fill(big_M, length(neurons(layer))) + ub_s = fill(big_M, length(neurons(layer))) println("\nLAYER $layer") if tighten_bounds - for neuron in 1:neuron_count[layer] - print("$neuron ") - ub_x[neuron], ub_s[neuron] = calculate_bounds(jump_model, layer, neuron, W, b, neurons) - end + bounds = map(neuron -> calculate_bounds(copy_model(jump_model), layer, neuron, W, b, neurons), neurons(layer)) + ub_x, ub_s = [bound[1] for bound in bounds], [bound[2] for bound in bounds] end for neuron in 1:neuron_count[layer] @@ -66,30 +61,11 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect end # output layer - bounds = [calculate_bounds(jump_model, K, neuron, W, b, neurons) for neuron in neurons(K)] - bounds_x[K] = map(pair -> pair[1], bounds) - bounds_s[K] = map(pair -> pair[2], bounds) @constraint(jump_model, [neuron in 1:neuron_count[K]], x[K, neuron] == b[K][neuron] + sum(W[K][neuron, i] * x[K-1, i] for i in neurons(K-1))) return jump_model, bounds_x, bounds_s end -function calculate_bounds(model::JuMP.Model, layer, neuron, W, b, neurons) - - @objective(model, Max, b[layer][neuron] + sum(W[layer][neuron, i] * model[:x][layer-1, i] for i in neurons(layer-1))) - optimize!(model) - @assert primal_status(model) == MOI.FEASIBLE_POINT "No solution found in time limit." - - ub_x = max(objective_bound(model), 0.0) - - set_objective_sense(model, MIN_SENSE) - optimize!(model) - @assert primal_status(model) == MOI.FEASIBLE_POINT "No solution found in time limit." - ub_s = max(-objective_bound(model), 0.0) - - return ub_x, ub_s -end - function forward_pass!(jump_model::JuMP.Model, input::Vector{Float32}) @assert length(input) == length(jump_model[:x][0, :]) "Incorrect input length." diff --git a/src/neural_networks/bound_tightening_serra_parallel.jl b/src/neural_networks/bound_tightening_serra_parallel.jl index b73fac9..1abd2e4 100644 --- a/src/neural_networks/bound_tightening_serra_parallel.jl +++ b/src/neural_networks/bound_tightening_serra_parallel.jl @@ -1,77 +1,20 @@ -using BSON -using Flux using JuMP using Gurobi -using Distributed -using SharedArrays - -function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}; tighten_bounds=false) - - K = length(NN_model) # number of layers (input layer not included) - W = [Flux.params(NN_model)[2*k-1] for k in 1:K] - b = [Flux.params(NN_model)[2*k] for k in 1:K] - - input_length = Int((length(W[1]) / length(b[1]))) - neuron_count = [length(b[k]) for k in eachindex(b)] - neurons(layer) = layer == 0 ? [i for i in 1:input_length] : [i for i in 1:neuron_count[layer]] - - @assert input_length == length(init_ub) == length(init_lb) "Initial bounds arrays must be the same length as the input layer" - - # build model up to second layer - jump_model = Model() - set_optimizer(jump_model, () -> Gurobi.Optimizer(ENV[myid()])) - set_silent(jump_model) - - @variable(jump_model, x[layer = 0:K, neurons(layer)]) - @variable(jump_model, s[layer = 0:K, neurons(layer)]) - @variable(jump_model, z[layer = 0:K, neurons(layer)]) - - @constraint(jump_model, [j = 1:input_length], x[0, j] <= init_ub[j]) - @constraint(jump_model, [j = 1:input_length], x[0, j] >= init_lb[j]) - - bounds_x = Vector{Vector}(undef, K) - bounds_s = Vector{Vector}(undef, K) - - for layer in 1:K # hidden layers to output layer - second layer and up - - ub_x = fill(1000.0, length(neurons(layer))) |> SharedArray - ub_s = fill(1000.0, length(neurons(layer))) |> SharedArray - - if tighten_bounds - @sync @distributed for neuron in 1:neuron_count[layer] - model = copy_model(jump_model) - ub_x[neuron], ub_s[neuron] = calculate_bounds(model, layer, neuron, W, b, neurons) - end - end - - for neuron in 1:neuron_count[layer] - - @constraint(jump_model, x[layer, neuron] >= 0) - @constraint(jump_model, s[layer, neuron] >= 0) - set_binary(z[layer, neuron]) - - @constraint(jump_model, x[layer, neuron] <= ub_x[neuron] * (1 - z[layer, neuron])) - @constraint(jump_model, s[layer, neuron] <= ub_s[neuron] * z[layer, neuron]) - - @constraint(jump_model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1, i] for i in neurons(layer-1))) - - end - - bounds_x[layer] = ub_x - bounds_s[layer] = ub_s - end - - return jump_model, bounds_x, bounds_s -end function copy_model(input_model) model = copy(input_model) - set_optimizer(model, () -> Gurobi.Optimizer(ENV[myid()])) - set_silent(model) - set_attribute(model, "TimeLimit", 0.25) + set_solver_params!(model) return model end +function set_solver_params!(model; threads=0, relax=false, time_limit=0) + set_optimizer(model, () -> Gurobi.Optimizer(gurobi_env[myid()])) + set_silent(model) + #set_attribute(model, "Threads", 1) + #relax_integrality(model) + #set_attribute(model, "TimeLimit", 0.25) +end + function calculate_bounds(model::JuMP.Model, layer, neuron, W, b, neurons) @objective(model, Max, b[layer][neuron] + sum(W[layer][neuron, i] * model[:x][layer-1, i] for i in neurons(layer-1))) @@ -85,16 +28,7 @@ function calculate_bounds(model::JuMP.Model, layer, neuron, W, b, neurons) @assert primal_status(model) == MOI.FEASIBLE_POINT "No solution found in time limit." ub_s = max(-objective_bound(model), 0.0) - return ub_x, ub_s -end - -function forward_pass!(jump_model::JuMP.Model, input::Vector{Float32}) - @assert length(input) == length(jump_model[:x][0, :]) "Incorrect input length." - - [fix(jump_model[:x][0, i], input[i], force=true) for i in eachindex(input)] - optimize!(jump_model) + println("$neuron ") - (last_layer, outputs) = maximum(keys(jump_model[:x].data)) - result = value.(jump_model[:x][last_layer, :]) - return [result[i] for i in 1:outputs] + return ub_x, ub_s end \ No newline at end of file From e24971df48f626317290c68f1c5e69ec8ccbce53 Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Mon, 29 Jan 2024 16:50:44 +0200 Subject: [PATCH 11/32] clarified naming and added global solve state variables --- src/neural_networks/NN_test_serra.jl | 34 +++++++------------ src/neural_networks/bound_tightening_serra.jl | 21 +++++------- .../bound_tightening_serra_parallel.jl | 20 +++++------ 3 files changed, 31 insertions(+), 44 deletions(-) diff --git a/src/neural_networks/NN_test_serra.jl b/src/neural_networks/NN_test_serra.jl index 10c4e59..1aa6d57 100644 --- a/src/neural_networks/NN_test_serra.jl +++ b/src/neural_networks/NN_test_serra.jl @@ -1,6 +1,3 @@ -init_ub = [1.0, 1.0] -init_lb = [-1.0, -1.0] - data = rand(Float32, (2, 1000)) .- 0.5f0; x_train = data[:, 1:750]; y_train = [sum(x_train[:, col].^2) for col in 1:750]; @@ -8,12 +5,15 @@ y_train = [sum(x_train[:, col].^2) for col in 1:750]; using Distributed include("bound_tightening_serra.jl") -include("bound_tightening_serra_parallel.jl") addprocs(7) @everywhere include("bound_tightening_serra_parallel.jl") -@everywhere gurobi_env = [Gurobi.Env() for i in 1:8]; -gurobi_env = [Gurobi.Env() for i in 1:8]; + +@everywhere GUROBI_ENV = [Gurobi.Env() for i in 1:nprocs()]; +@everywhere SILENT = true; +@everywhere LIMIT = 0; +@everywhere RELAX = false; +@everywhere THREADS = 0; using Random Random.seed!(1234); @@ -21,30 +21,20 @@ Random.seed!(1234); model = Chain( Dense(2 => 10, relu), Dense(10 => 30, relu), + #Dense(30 => 30, relu), Dense(30 => 20, relu), Dense(20 => 5, relu), Dense(5 => 1) ); -@time jump_model, bounds_x, bounds_s = NN_to_MIP(model, init_ub, init_lb; tighten_bounds=true); -jump_model -bounds_x -bounds_s -@time [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750]; +@time jump_model, upper_bounds, lower_bounds = NN_to_MIP(model, [1.0, 1.0], [-1.0, -1.0]; tighten_bounds=true); vec(model(x_train)) ≈ [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750] include("bound_tightening.jl") -@time U, L = bound_tightening(model, [i<=2 ? 1.0 : 1000.0 for i in 1:2+10+30+20+5+1], [i<=2 ? -1.0 : -1000.0 for i in 1:2+10+30+20+5+1]) +n_neurons = 2 + sum(map(x -> length(x), [Flux.params(model)[2*k] for k in 1:length(model)])); +@time U, L = bound_tightening(model, [i<=2 ? 1.0 : 1000.0 for i in 1:n_neurons], [i<=2 ? -1.0 : -1000.0 for i in 1:n_neurons]) using Plots -plot(collect(Iterators.flatten(bounds_x))) -plot!(collect(Iterators.flatten(bounds_s))) -plot(collect(Iterators.flatten(bounds_x)) .- U[3:end-1]) - -@time jump_model, bounds_x, bounds_s = NN_to_MIP(model, init_ub, init_lb; tighten_bounds=false); -jump_model -bounds_x -bounds_s -@time [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750]; -vec(model(x_train)) ≈ [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750] \ No newline at end of file +plot(collect(Iterators.flatten(upper_bounds)) .- U[3:end-1]) +plot!(collect(Iterators.flatten(lower_bounds)) .+ L[3:end-1]) \ No newline at end of file diff --git a/src/neural_networks/bound_tightening_serra.jl b/src/neural_networks/bound_tightening_serra.jl index 1914176..281d0d1 100644 --- a/src/neural_networks/bound_tightening_serra.jl +++ b/src/neural_networks/bound_tightening_serra.jl @@ -28,19 +28,19 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect @constraint(jump_model, [j = 1:input_length], x[0, j] <= init_ub[j]) @constraint(jump_model, [j = 1:input_length], x[0, j] >= init_lb[j]) - bounds_x = Vector{Vector}(undef, K-1) - bounds_s = Vector{Vector}(undef, K-1) + bounds_U = Vector{Vector}(undef, K-1) + bounds_L = Vector{Vector}(undef, K-1) for layer in 1:K-1 # hidden layers - ub_x = fill(big_M, length(neurons(layer))) - ub_s = fill(big_M, length(neurons(layer))) + bounds_U[layer] = fill(big_M, length(neurons(layer))) + bounds_L[layer] = fill(big_M, length(neurons(layer))) println("\nLAYER $layer") if tighten_bounds - bounds = map(neuron -> calculate_bounds(copy_model(jump_model), layer, neuron, W, b, neurons), neurons(layer)) - ub_x, ub_s = [bound[1] for bound in bounds], [bound[2] for bound in bounds] + bounds = pmap(neuron -> calculate_bounds(copy_model(jump_model), layer, neuron, W, b, neurons), neurons(layer)) + bounds_U[layer], bounds_L[layer] = [bound[1] for bound in bounds], [bound[2] for bound in bounds] end for neuron in 1:neuron_count[layer] @@ -49,21 +49,18 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect @constraint(jump_model, s[layer, neuron] >= 0) set_binary(z[layer, neuron]) - @constraint(jump_model, x[layer, neuron] <= ub_x[neuron] * (1 - z[layer, neuron])) - @constraint(jump_model, s[layer, neuron] <= ub_s[neuron] * z[layer, neuron]) + @constraint(jump_model, x[layer, neuron] <= bounds_U[layer][neuron] * (1 - z[layer, neuron])) + @constraint(jump_model, s[layer, neuron] <= bounds_L[layer][neuron] * z[layer, neuron]) @constraint(jump_model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1, i] for i in neurons(layer-1))) end - - bounds_x[layer] = ub_x - bounds_s[layer] = ub_s end # output layer @constraint(jump_model, [neuron in 1:neuron_count[K]], x[K, neuron] == b[K][neuron] + sum(W[K][neuron, i] * x[K-1, i] for i in neurons(K-1))) - return jump_model, bounds_x, bounds_s + return jump_model, bounds_U, bounds_L end function forward_pass!(jump_model::JuMP.Model, input::Vector{Float32}) diff --git a/src/neural_networks/bound_tightening_serra_parallel.jl b/src/neural_networks/bound_tightening_serra_parallel.jl index 1abd2e4..9d59946 100644 --- a/src/neural_networks/bound_tightening_serra_parallel.jl +++ b/src/neural_networks/bound_tightening_serra_parallel.jl @@ -7,12 +7,12 @@ function copy_model(input_model) return model end -function set_solver_params!(model; threads=0, relax=false, time_limit=0) - set_optimizer(model, () -> Gurobi.Optimizer(gurobi_env[myid()])) - set_silent(model) - #set_attribute(model, "Threads", 1) - #relax_integrality(model) - #set_attribute(model, "TimeLimit", 0.25) +function set_solver_params!(model) + set_optimizer(model, () -> Gurobi.Optimizer(GUROBI_ENV[myid()])) + SILENT && set_silent(model) + THREADS != 0 && set_attribute(model, "Threads", THREADS) + RELAX && relax_integrality(model) + LIMIT != 0 && set_attribute(model, "TimeLimit", LIMIT) end function calculate_bounds(model::JuMP.Model, layer, neuron, W, b, neurons) @@ -21,14 +21,14 @@ function calculate_bounds(model::JuMP.Model, layer, neuron, W, b, neurons) optimize!(model) @assert primal_status(model) == MOI.FEASIBLE_POINT "No solution found in time limit." - ub_x = max(objective_bound(model), 0.0) + upper_bound = max(objective_bound(model), 0.0) set_objective_sense(model, MIN_SENSE) optimize!(model) @assert primal_status(model) == MOI.FEASIBLE_POINT "No solution found in time limit." - ub_s = max(-objective_bound(model), 0.0) + lower_bound = max(-objective_bound(model), 0.0) - println("$neuron ") + println("Neuron: $neuron") - return ub_x, ub_s + return upper_bound, lower_bound end \ No newline at end of file From 8c68b86126743f52005cc2e08f93d7f843a71425 Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Mon, 29 Jan 2024 17:18:48 +0200 Subject: [PATCH 12/32] removed unnecessary files --- src/neural_networks/NN_large.bson | Bin 39253 -> 0 bytes src/neural_networks/NN_medium.bson | Bin 11308 -> 0 bytes src/neural_networks/NN_paraboloid.bson | Bin 4668 -> 0 bytes src/neural_networks/NN_test.jl | 33 - src/neural_networks/NN_test.txt | 56 -- src/neural_networks/NN_test_serra.jl | 4 +- src/neural_networks/NN_test_serra_parallel.jl | 31 - .../bound_tightening_broken.jl | 677 ------------------ src/neural_networks/bound_tightening_new.jl | 162 ----- .../pretrained_model_pruner.jl | 7 +- 10 files changed, 8 insertions(+), 962 deletions(-) delete mode 100644 src/neural_networks/NN_large.bson delete mode 100644 src/neural_networks/NN_medium.bson delete mode 100644 src/neural_networks/NN_paraboloid.bson delete mode 100644 src/neural_networks/NN_test.jl delete mode 100644 src/neural_networks/NN_test.txt delete mode 100644 src/neural_networks/NN_test_serra_parallel.jl delete mode 100644 src/neural_networks/bound_tightening_broken.jl delete mode 100644 src/neural_networks/bound_tightening_new.jl diff --git a/src/neural_networks/NN_large.bson b/src/neural_networks/NN_large.bson deleted file mode 100644 index 45166e9b41b142505640ee1830fbe4619ca5e5cb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 39253 zcmeFYcUV=;vo5&FL2?$5EJy|s5Li_pQ9zU+K`^6Y!i1Ouf)T{5DCYEGz=#o6m0%V` zP)wj0Q4|x30VA`&Uzq1SbM8HV&D=ZBndv|FO1)Qi*Q%=Se(SCHO%jP*`os|<$4eqt zNhGpUhowoBBoax+)G3*%Qzdd!XHFU^F_-_>6BX%`5yPeq`^R1RNyDZLOV5xzl#{-~ zO>$d$_{ZP>hm}_Uv(gjk3;ySowo0E$-xU5|R#K2YmB>#RmM-nr-xb{?iqb>hzT?LX zm&m&R%Xh~Jt$@A$?##R0-?jeL zq?zeT{}4U?+Yj+St8>uA zDI@=_&iykdjsIJn|8uSXsPsXjhmD!=ccs5wf~3>(e;BWv^zaY$|Jy33{^LFWeEvG` zAJ2dO+fDo@FY`qDhyTuo^G`ASko|`kvSj{)n1)T6GHm9*d~L*kDTV*w{4D;i_)kyt z?@#x)j|&|?aoALk?$R^)j4`uDN=&3@I&PA`{np<_$6X@rz(0$^KfYV*-xr^^F2m0= zHd0COa+X;QXtVz~@~;hGzw9{AyS0&fmTPja(#2Tp-Vd*@Sj;9yRgi4DQ@o{lm6}_M z`D^-LLPx?vl698|4&W*fASM(NgMhPpFr|YxzG4p?*H40bkdtBZTuhO|4)|qf7=B7 zPnG|N(n=eX%3RLlf7a%cfA{9U>9qMzQOHUSQ@X;LC;x~au%$K>6(XxZ#;Sr09~=a1m?YX4f1#wnD4gyT$%3Z=>Yn_YpDqY+G< zFsXgGkfE!=xqj`0jhBQBmmXZB;>x=DeXx&}Jm+*BP7lO2WN`lvwc6Nl&y9-Q;2G%K z=Xed}Kk%W=`u4cnqDHhR2;`?;XK7fZA)e}b7$P;5aoN2NnDasrzpcL@#B@?;hxKQ~ z2Lnv;d(Cv1-H7ZNyBR|7KBm~#jj$x!54(<1M8nO-@X_Wtn4W7P+ao48=+%3_((Yy$ zv2!t;nq`BE4J)C%EfAaQJ=tbk2)fuC@lfY?!qwDIFmS57@Qvm9z6tL|%ED<7T|Dp`I5PlW%sGd%? zM&ryj4eaxUXzl_#Y%uDA+t-BRpwJ#XBRmQ7f6o$J?RLPlVVi~7x9*Zd%2IJ$Lo20> zP-M$7D`?Dsg(MT*2?w0j=00C%!pZi#@cH9IQ2VZoWuCj>=)(k%RZQmevE5Kh#tUQp z+%eHG3C6tGLb^})(_E?}t#j)IjCWw|Ua7*dla5@P<&UNI%A|6xj0#7{z`k?pywvrh zuxHIVs1I<&rIWnbq|k>y2xjc3Sx=sJ5&XN4HhKmrAv{#zSzois+GdGZZ+QVWw`uW6 z<0ImU_O}#X8h`;=!(a&P6K8+hS@vqrE^(Zv36H&gRLE@FMV{$9=^9H_yJBaU zd44@ydAQl{`bTf6tScz+X_4q?{2R`XGNChnb zw6nsIlatsjYCmZO#N+tRk74SRJV?7dfR{|{gWkOzP*c|u)?Ew54Vz;4LQ*#{-7DcY zt_ds~asw7>929+Ke*)*iFq}JODRuhdfN3u7aJf2!AB4%^*(D*IoU{mjE&Bo5@7F@# z2YR@uPZzxY=MNon(ZXoW{@f6l2Oie)c<_{+u(%iU`A;>Fb5IX`*JRM~GZB0<*PA!E zf2BEnmcZDHYWS>^9oLtQfOUn}{TvoNfSWfHID42oyfcyEK)e7!o7&)FqZLj$6wMPB zlV5mC46i+@z#pJFu)w zKQtU{ghsQ1sc3={ZjMjJZ^PfgNEK7giB^L#%^HHqCQtaSZG}DE)&Qh`5L;6(Lcb3_ zIC|0virHx^W*47?3f~W6ZJ{=sSa!lAc17^6{sr0JO2nbV-FbQ9C6INzPd7)E!^LSj zbneYN*rC`TA1JS&G_wGf%+bN7uJZVG@pzK-TL~%}w&?ir67}8DPq@;5BYpa2BV;$M z7m5|*xOCoPaJpxLGrq5&$_Ksq@-%HOS*47%F3G~)gSTn#`x1g?dxT!gC?%_!;+xJ< zx^^-fFZc?BXLrHpeZq0J(l}VTPLV@hchUWEeb^u@h;4l(7{O7z@24>z{{D!XHrS#@ zdzBa#dzBK49I)Bq?I))E88BSdX7{ zvtk{?5@FgZ4a^xCj?2BK!anyGu)*Q5Fn8%1c-?SE7&z8KxE;I^HYj%IH{Xc5jrHOC z8-m!rV*xxmb`geXPXLY1UrA&4XIK>d3O1~N;48m-q4<5UIXCt-Va)v%REj)T{8qrC`Dh%i3{#^)Kf-%{aqQ)O_} z={QapdxJ*2Zlu{S6EL;rF9?5h3|5S+gKY8`~_lLCm2hKh@4|oYEx2y2Syw??`X9A1#kt zeYM9DqB;I~sCt3``i z#X2#@(wFLU7sBYN6QKL|7<%6HnM}XNa_(kj&dSzewIk~2&_jbg@i#rY{2c6O_2651 zx5QI{x8cz|MLaOfk)ORiN8`rGV)Aiy?(5N;_eDGSZFzKB9C0OwjLKKQ#UEDaIX({i z@0~7c=586osSqYz|4glaJn8V3 zMp$xotzaK1ig#9S^xN5Siy}WQhf`hD`Ps^F{Bp8_rj2rh9igkqQB>nIYoRj+F5rchxQDvJ3Uu|^9!>>Znxa278vsq48oL_=w)L7W3-IJ#lTVvBGrtY`P zU=KOr;3Q{y+V+PG=J&zNy_5M_rVZBQ*ppmEEnV}x0Gm24fbH&heljN?6k1MGh^ro( z{WwAg$2g&rgFUzmT1M9&-V@}8_UFSDC!qR3544&1UF_TK6!mBi$G|FYPP?#B*y&-* zbs_oi=V~M_>SD<_LIvjVzxM7j#nkRA^X>749;eq(>S_bQx=IUO z>*m1>wHgRGql>v{gb%#C@OQf~EKvshyj2HPcBf!n_m4E=-3K~!L7n6MP1){PKb}`+ z02l5p2AOB~=ubic>2fdJ{eFvJHSsDfcx28eSN7)f1MgFAqb(kNoG#9d3Zb3Oh4kW_ zH&*cj$}=_Qi{*}-@Ky=7pE?F*D^7#=o=rm1+E|{{V8$-KF;uo*moy7QaOYANzB%VP zSZJJqpR#jl`~-Wf?0y8=%HGrGF&eCv)r&$__r*_c18{rUW17G532c^`MJYqylIt5C z?DF9wq&RqrsZC+5tGiOXxVTa1TIWN9JH3KkH}})QxzRXT!5Q}#_2j5I$=J7_1s|Tg z6Ha<=qoI3@XxXn&toq`FuRj{%3D;+k|7nykDAFG-Ve$}xWAHAhX&)MUzVs}drM4n)esNQT1Ff_ zgm%kCjZ(?SED=I(`~+DfQmmpYT~X^K53&#uQ;h_hE2;*rsYxbD*e@pM!) zM<4IbwL7xK4b>m%ynY<^r6gQi6N~ej7SrP#2P_<0O*4w4(EHO2S}@=>{7IByGwEJL zAt_Y+svZySsEz@t@@UjiPcAZZ#OSL5bXKi`;&%T8y}CE#<$4Z=dnT~@`u&2tjv<=G zXVKQ<3G7KhDC>O$M*MmNfiG)d@e3n(lCYP=NiJx4${5?{EvKOKsnC+;jopt+{YSQR zPLj-`#r-zJhz{A5-)$2lTRe4zHT}@=r@O&S>|Al;iD! z;;;);ezz+}jrcA8@;e0K;Ykqg*_(R|RA)WqJ{%XRgcJHmfL~96h4qi)E}2_lCFu|!r{>TQ-aq1JbG?xil3|l(f3;bsT&z!`hG_&O;A7=bq$Qz zWePL618OV6@qPExH1w4N-$8v+uPnv04 zNN-@4SXdT8#cM~9owhTdd3K2o@AZL~G2KvkJBUF(8{o!gcTngmhtGemq>7vnmiO=_ zt=J3DJ=Krwx~`^~Ka}vKY%jLDagTBaZ-Ye-+hCZS3U+o{OkdrjNxR!eSeapfr4Jp- z8rpjD6ms;%&~;?|vnRFYwNmM{PvWmQU#@DF;jobbY~1fL;QEWA>c|4PyETfJFX_ci zH5NP~#fE!$=tFg8H0zmX!=9cpFlmcDH>jlGjN@G~LdPF7@;?iYt;WLmr=!oV)M$ez zJAC1WK`*{wX3m3bI^g<~-(|ZjY)hB-ze5H#cW7_dXRy=mDd-;e0H4F#c}xgz{u9MldMM)j@*b@JAPk2uxCP#A{I2v29%2L84^!r4wWq~p%YyOr)i&Yluft;X$N{_| z-H4aZxklG}hH$4n@*I(^2ju~O385<8=w{AYFxQ-V}+D^ z--dHW1!Bn@AHH_#J}J+$rD0E_Fukk*hV*?#EvFTU`#Io+Cl>Uf!x#IUj`NET{8;t~ zZ9(U_3kGhU2Gzl)Y~ZfNpDiv!|MfPQ`mzWb%GzLaQZQ^S|4oy(Xkw9XF==xfS?p;B z8`D!X_`@S{MZa9Qc)t_A7;M7Nc6xxqOa)k))L+ci$@EK2*-m@*4nWx(n)qjh84s;K z0Tt^^V5O%!X4amAmVw*Ib)78Abm@gzsu$t%l;;$XvR7Q~HnvRdl`(7X_ZIixy-#a? zuO^+RhFGGdB0{i$Gd?%LrdBz+Hrf*Hc1H1;dkMH+F$L!|NbrHD8NHuYN|(kTfctJC zT;6*G9IanLqw=kJXXX|-v!s?n6=P}Y^d<1^qQDvX8my6}$2;2>L9A{rq@CywmOpy( zug5d#gVoQns!!@Tt9Awa3{4hJ8P|iPz@2VrGIXfDh8#vY7L>$i@Q=kc^95{emXqguEbA!ShLZd&Zs@p3NMd)K$zI>HMLTZjM zW&e8WyCI&PzlLJ1h@3yq5i~{?!-VS36q22YDH`2z#<6kq5&Ph&yU9G^x+71P`hY?4 zKZO%5y68Q~4pjU@Ff6dU&mi@7*w9ywtM=+*m8UJ!mksdZfFE~Lu7H&BOdTCYJbsJ} zW-c;DmC|wI_#j2l?|4af-g;q@oju0hiG{=cl1T5*9kB1{fRN6;&@3>V7p`*SrG~O# zuPBGZ#=ipEzK;?krF#bD9$fg_OS<;W5ex495%wt>;|>VqZLKfiRLUEuE(l@M-CEob z6M**iC&+PAKW6JUf6SH^@3+JZnKg9jdKk|1TSDO;Nf>_j zBPmEUdBYQ1HZa;vZH^Wo8#0sT>4!rOrNXZKx1`^83U=1n;*aqX zQQx;W*!S|mjD>A}qlY*8t&an2RMAAkDaq`U5`p38En!~!1-f#1GHeXG45M| z8w^$|u_%{2RoM8KGi>$^!NYfb`9>CzaQtl59JGA_20Kd=-$K+lN4^K^{&s(3tfu1UuIIkVn$25Y* zj5E+YG7CI3tXQF11{a>u<}FgZb5>hd4mo<7rl#hBPSEx;y?_CHQdg6`%u|FXzn9Wi zTn;+PPJC?IY|0-07ShBvNkgDI?9V1U!-euUTOo`Uh$ zBk<|7r|{<58Q6BbHy<(W4ArkXLFL5GxU*|dd}wCOp_R*E%>3V!eJ2Dv_IGA4H!FS} z+76;PlDc$CfYGxwP&Y#zPN%OSwPH&ea5S9Gw{&K=U%5g=Kja>8i=560ob$s3I}{px z3(9?Q&6lt?>_r@_2FEkeZ6QsH6YEsERMkCQhqqKnS^ps;K> zS)U2v#tB;VWA_x=c<&vooBU8v{;`BgH@p|DU00CPrd^cawj0)J$)-+8Z=;`EPG=HddePCk^6df4zW4=-NsIa!$5 zshJK>jKW`qUHMVV0TS))ap@W*R>*#zgVkd0WJtGGVx8OA5UBD324v zl6YNRH!OU5ms*>TLWZwCPg!J4F5enS)h>oFyO?nN`Cfc~*cos=<;#&f6L3n1I~Xaw zpxH+1Jn{Bku+Ldcd1cO&e=L-`Y&<21#v$Of$%IE_T6594zO4U-Y3h?g3UhG8)_@E$ z2%1lio8+C*oSAjiS3&6!&Y6hMq(5dP7*%U>$0ZY4b@je_;5oUC?aifZmo$Y#yho&p}=ErTN_+LG9IN;>=$tS+<`zBp3SMk4HBh6L5+j0{TvGfR7omOwBcVy38I&j+9~Xbv*kmT|{09 zI((wnJ9t^$gHBO2_BmTGJ`WF~6HAvvwuKJVdqi{bkmIl;;a$&w#EWozM+}EN^yRPq zekh##B-*SBLm9i*5IFmTxV6xXDmn$@pxiF#U}A+kmucgr8v*Q*RYS2=o5g`olzk%Y zwYW;v8C}!V*kav8@^6<=veE;(DgO{o#7EQP&7IkHTPOz%X(gN9U#ai7^N`!o6^A50 z0gue5BS$mmB%~>(!)LKw=3})@0MyQZ%fxq0E#HY21ynM}CS~Q^*Op><3;@mC5X8V0* z1C`%{%9=&sFs?ga*6)PjpZk+xbr28#vIu6py-sGln9}FI1qu$}ZQ7G4bj1PKzt$BG z2Bz?az5%S~Gflv!_Sj{a8tPZ6^Y(-#!m|n=RyGRY&E0=N^r+o}K0gqzt2y%6y@v(W zbvFFxdMhQI_T=uHKGB7Ts`z}AGRr?!!j+*nY0s!|PIYV-T-7f^LhC2E^YSpfGW(m4tM~$F8QBF<+hyiVe_KB@xGUY!=2F ze-P8=^uVoOjd6R1K002E!+hx;ZPsTA8g`B1+1O6CgRL_fWrxKcFxv ziZ$F#uy}10cCu9GzwTY5VS*DrNnQkA%XP8qjYiT>@5|$^#bWR#DK64K0?*A7_?b=* z@b4eT%huNl*RDOL*?+!3_NY(7%*ipF|8W4jK39dWO-4L<@KQ<=FM*D94Rl3*0It@Z z2@O3$*~=Di^kGLXR*i-u2bK8aB?U}8A&-V8DHt0UfoXLPC>&LUZ5O)0&~5=-eKnYl z2aFWg&eDR!p^sqC*UY0TqQ4B%ODS2TaIv22cAHvCR@;olyW&~5n}s2y&>k0W=2 zlX7<1Ut`|V=xK%+lJJ|x%+=v1B;!V%v5ZBF|EjFzsrNI8E ztUIOC0YtpxvcA2B%B?(<)OQm{vg1hnJI0y&5j|Orn}A zgTzkD=g`PnH}=V{pu)KCFy%!I?zc>2k1LLRe}XEWwD!i{k6XlmJgF}p;*15c`WSVy z3s3jgDm}Mo5Lu7Sq~LxLoVzLzu-*)NDm|s<(>uYpPpsc8!(J4ZF2z~3w}QnsS-&NB zUO__CGeI7EqL#`a%J*-Du%#yaa(Oae8{Y&ea<$a-`wMudd>3~I9~D;ajl}dw1wOXb zpWY1mEKWOYL36%77NX7ilkxbk^!$lEe&6ehR%x=>wMKbP;vCSP3x~ zw+TzMw1o6SeR%5+7`+7!0CV#-BM z6QH`v4d2XcfnyhgghdmRalx1gLdeS)R#@oEx91ph#bygEvakXD*b=a~`z>i>}7pk`z3Qhn?9x2Yg4|9H;-+42Zlw8JhP`8jlP!yMPP(^@@_olY#{d-Cc_@x zHK`=VoSSFba(@d|=+?w=x_<&YE$oaBPYW1x)`^=WCqUc73I2EmDjR+g`4|1XX@-@JalEiGhia0JLAN-2PM$47AGfbm)AOR= zNQ)zIV!{Jb*0+UA-%L0v#*ocyOgLZy(dZ(2p(yTI}^R|lUfW4yBY*f6II-= zIe-V{Cqc(q3BFhw2df@w-~r87l(b+lEZd}x0|%TH-#jg+s*@J1{jf8S_$-h8?e~J* z84aE}S%KfI%@AIM^#!>hd8GUN4i)HK5r+#&IB_fn_^r&Fb|++nfgwJFAf-bw=#;sgh!AO5wQE7IFK!-Owg) z&ih7(VJo-8y2F*=7wduFGSxU?d<=dZkpP=Ya!JwTi%_Wg8NM_fp#k^3gp|`EEcvhy zc2smh$>UITxn;?#9{r>xTKC9&KnFC$>GSjVZIC%36knAo@P@UD*rs|C#{Bs~vJ;)5 zS>_rz%_6wjaa@$snF%MN9tleq-Gijmg|umlKX)D03aOjTnab_hhWRQ9AsT@SQ8 zsL8J^$HAB*dE)lbu{^+9I>%|qkRn<0vI;|3amRq~r+V`I&3%`drHq!_(#!ns^?I<~`=c6({!p@~{71iEuS3pIE(NV;CHZlsJL4cw^Gi=NUB zGKw(AQ7^jU<v$dd-jW~vxkG@n);a;L>65VG(iF;R# z7H4Y2asBlq*!4hJ-1N~2ruTN?9Z$lU4usRTVgu;rsU;2>lnT|#>5y=?6V6vSR5tHJ zC)|C(AI{$2MbQ)4&pWJ@jNiV7g7Z_P`%q&}I&hMzE5E|A6PL*LXCTVD6@j?+p->{l zGGn^DBd}{H6O}d4DSR@i1~?0&UY;X`ZpePaF2f;b1FT5X!NU?77B3n^@!@Pc@y^%I z+^hIJ=`Ni|Yrb@1Z&L}{`b%xJwhmI>KBEsXT3p;`2-xrGg(=frSY`A);lQ))r2F^= z_?}E+nQ6PoY_<;?4V9x%%TDyQQ@=9RrLX8$o;=T-ZjbKXCb*@)E|<#}lqt@62&VUn z%9h>P4jy|Fafqr6`?YlC58EF@m;NirWZ`Sk?W_#8qzt8|PC0b;sW#ux{z&uPJH(5q zfO(w*aL>00UM`Je|Jh@Q9rsp%m&Om;xOF2H^-I8_%hSYBy?^=0Jk-EO8y#HSp2T7A z6Cr%CJ2iV|K=gJgc9wh@@}sh8olI|fut*J`KJAVV`WoVu@<5(qk?rSS3OFsKiZ)C- z4R$8Bc(<+_TwAWd%TI=KLWeP{dI>zs=PJdO=F<6(vBIE6v^@PxgY>o3Tg2iz7v7#|b+Z>xb*y(Yq(YZJ&@*@>4bwUWjLOOT5mEaaF;xnR?Lc+%|! zO6{jD+C+6QA#1+d%H6ukc<5hgx7K(}m4C@0?w-_}Yw3f-c4 zY^o*S>lTQ64G%(e!dv<}+mXlgjN^OS`^yI2(8A(xpW#QRPUP@cJ=`e^2HmlU8h#t; zzdQ0OtM+_m7;gvni87sqvFJ@Dt--@z<975`6w0KJ=Q?lB2lP<5n zLJyuwv5HH_g&5Z+@lN$mcpGz;Oy&h)wW}STj!)t@yOdyRuN;d0t6H3g(s-_d0dxtH zNHJeSe0>P{c%2&M4wUW#?l+4&(ySoprvvSpz7X6t2V&pF^0@n^yr6b_GilqG!cprM zI6X6(^mLy?(FQ3ll=VRL3Co~8b>?VobBp?Jx5j0+=7=-e8=>ZEGWNdx3w9UF;Ot{k z?%VZ?)Jt1iv|IKG_n&&JW*Rs zeV?oqlHXN<;jjLD+4h%kK{*nGzITN=mg<;Ux{SU-jGtvz8@bN4#uJ4xT=V!P1s;tC z)n7sE*!c-uyAXimex``EcMC}`)F0*4o9M`U3590Z^N_9i;@K63e7nb6D1SGQdi=GR z9$2-~w>xjibWR6ta{egB6#LTDs|w^@`A5*Z=*ZbyE%8fJI9sg>5U0D?mG+C&q~$eH z=r`Y1Oz-cb@jxkEh2<9uWIZ6YZx=CeM4m zeEFLPZatdFD^+`8!lp#g)38m{iQWX&mh})}dX#>dKZ2oyZ}|30{|Ya<$8h{L7yQ&O z1aV^zPTSrSL#_I950}+q+OqM`EAu_nCoCtiBoFE?lu*)QH}=o16dyzboL0>Pzwv%N z6Ra>wCXe#m)G>LM2PxwQ>ay6FH+9$G0$nru?3fO(4_l#Ny_Y8J?TLr%Pr;j&GB{S{ z4}ALY0(9~{c>2*Duw=Isce%VCp1qNB;-ZwfXKn;ujwpbJNy|XtWg|(~U&0SzNSRM# zIDlrbIDCRTD`&;Rx+QPLwb8YHJuGc`nV&IdUopby*MD)5 zC`R6I0oeR@0ivN%&&5ET7#I zfh36$s>Tiz!Y&z;jTH0Gl&hfPiam5EUIv$ps)3VFa|N?tO$-zMBG=VjC^1zU<9gT% zdVO`cvdE36OL3W$yC>lC!#dCmGs61OCMfJFqoMo1L(9Tih^c%{AFa=W`Rh5bz1{?0 zeYS$xIo3?kv7EQ^1jW^Ov9ejF5Et)>n|tr1AJeA6wAOc0d$r|bhX>%OSsTcGfD2zT zlX9jsmr!Nmd|JAqp0xUga$=kfcGj12Tjx&|CTD%1s>{~+Ug0Fo3UtJ2gXF-=zz_Fa zlKQI$8d%Nf(W6Iy3BxD&u+z+Ul&RO7E;%2CRV0l=RL`Uk)qWV!?=O&| z0K&IrvHYb^08VP%Dt3-tLaRqTq)T}^Y&S-Np??(1PINm&&o`ZhoIAJ0AF$ZQ;e)>Bc~%-lw)L$Bd-U*r03m4uWS9q%boPt_HzeR*vsK2 zmj=IsZ5|kWzZ;IX$fW22W#X6%JN#_=sbQxUIiT>~RK#km`6v$*o(9q~Jtb7y5XA-M zefX_;G@h$5;`$*^;LgQybY*5KoviD@FT#RRLAM8-DD>y5g&thbyK*GJYkDDXtL1e{KQu=Y2WuWDu@@)(Riy$zVuff6!gM zQqal>6E+l07n;5-6u0#L;P>uYAl=E)pcCfP#n_yA(m2w7$QRawf~qF(AAAwUoy>N1N=_g4B%QN7$1wpeR5IU_w^V09UO;g10*O@ zepQ%fsI{L?7*zUHGtj6@`m(qB<@ewI^&?pSra!-ii zwFaoNsFEh;dGq^s?Q~=4ZfZ#h;(LzEp=xXtdMx?_Px%1IX9nTzDhpiMbcP=Fz9=l( zn#>2K+))<~BlI8RgRe6FkhW7Ev=&dKU(&evkAdd6ILsIWmJA{n!z(m#l^GaYgrlZb zV(FXV7yYi~+H$u{d3-Wp7>u1-1-*6iV2a{5aO>s>D`Zq5>rxcbhv)F9;e_~W$UZWy z+X*}IxVCM(#x%&ZqO?J$p1y# z&Aa31frz?l))-Zg3f_uZoSvkHiP{G2;an*(TeZ-0jvTK% zE@j%gS+Z~EWU8AS3_pJChDlmGDI?vFzx`|hyV7)!RPPg?nRe%u*oS~e$%kVs^ zGal;Z&2Dcb*d^~XB^%zM(v_>pZs&VQdmbt1>)LT#dnDxeZK8&94XTze_8Uvj;mU(6 z(ivTbqUsB!sO7@4ILK_3My_l%%)JsCW7L4ZS=7I@%?I@;+cfVy<&xw77ZYsLiO zI|~=4amrZP@fIW&T4LL$A&`3Aiu=hq^P72*+$v?r?wDYSC&K}atb@_`s;SWCnokX% zj)*_z>yvY|7OhtZ=cu`9;!5XRGZH+hID{#OcAds)vmx?-WLABL^GTry{ zVXcpn6k+!Riw~hRt|S-_)kvr%qZ?ir(g?nLx^cjk)3jCEe)HZzVu05OkciI(^+rVm zvtC@}^NF;i?5fNKPHZQ$5DLwsM2qz?wDguDX2C@&{9bo9Yu;Wslya7u+a5y8HhHde z=)%8WRf1z2VjX%zopTzMtRyj0%Mrt5d|_ADKD^_Z2`f3|LBPoG6x2A}Gb{{UuD;CCTxno%JQjnWeOp{u?d5~oUr%OIVL(ol%{Gy5GUjxv1 zLr?lPMIGyUwbJhT>lC!}IOU%o47(GmNb>S)S&79>A#K*oGKJTH{N(v9xN79ZFioBF z4;731Mk`~~u+9+Wd`moX)0nej^l7-23?E@Vt_xSdmiucVOMWQqcpneDx0-U?qW%!* z)B~H#L&ehf##r*_f}n5&xL~v^W^Hj6vMi5K*3(cb>c5)GU8EU62A5%y!!dFV4aK5O zZ@_Mx4{O>+qLzCL$$bds>hC6$HC_i5>@@gvFB?w&o-MT4A&xB5DTPT08KjO|toqG(AW>>lWdbvkk!^27m^`kkWb zk>4m!%FOaecwBbnaSIHsH$;n#%CN)I7_2{L!2@F{8u?4gj(nX89>FIlZ0ZTfk@ow} zb{Xzg&|8{Cpu|c)J0RlPSu!#-!0PO|W$L=^bg1PCw8bl8M7Ds&d&UZKhk~%APzA=` zErRA7eM!?CSwY(YC4Cb?{h%LRF}n$AGmrWmSX2YXQ!jv#vIUn03N9lKq&Z$uF9dnjeGs;hBH#A#}%nhfXcfISW-32h3`#Oh{U zYN_z!tY=bY!TlaAS#}0aAC_R)`5TbBu7=hQiJ_q*&G3r05q;aPf$En_ga|XljDc?e z)bpV9fdI)v&B!ATe6K9u1`Vb@6s12+aICiC!pt9}t{VnF%=~F@HehniDk0TE70)H; zpvS>l-_nFIGG22}Y`FdoBsOcH)S8eNzDg)Lc2_uH&_>l8D~0B7S>g{zS4{Wm3Z?Is)6vznbZ>M7X7BLD zd`(NxZHNJ_c!5ja-K4qFG#*LfAgGDc#OfJ(+&FFr^_=2|g*|(TLnqCH*#1sCSPb$S#mG zw55K%QObhJ=>|Cq8t8ds2N|zO2GztnbZAK!m+5(NR!|J&kM1DzR0AxO^%B>18%0&Y zCfH!63LcIF%BH_J#lkSRvUjwJR^B$mF8MO7UMFRC4RdG7$`IlAvv73oqYUZuyW`fv zPoxuVkEgG_CiS!vLgDcs;kceLFTEwlSN11E^GjJF|Arbpv0MkGjk0KL-I@-CMlfeRgOQ+D4q%^8gbHJoLdCa@#k5;Sn*!z1G zRHt?oqQXKb^P2`AxZ;F4qg656t}_R0lcQCeJE(5jc-URlOv$U(QBjj9NCLW{WN}Z- zxtK>qzZMCTUR;MX%h8k{R3#KXKQGF@vS;glJy};x10BlqXef1|246&rXUl1bL2rKY z=?2M;)Iz(&5GdTc%&&0y6mV8M2y2>@xyH(Y!+?i_^K0o~DKx(;PY zGr|r?ZjwjFHnFH|t&l!W2X1Y!M&n6}(2&*(l&mtK+1v%KtfcJrRSppJrZWfWyrOqD zvmw{Qk^AlF%Ie$RQsKGYBw35WMd}9bE-Sp#T(;xEH_DbW4h)wR z)0KDmQ0pwqRCojKEh&Qho2HcZZk=CA_C%^&^%{N~d*QBO%6Mo-F-0$YMyYqIXl>t4 z5G{??o!3^ts6}}caC;gZ3f@Fng-%d;$(7sc?OBp?hGvEAgE}cYK<=C~{+4FqtZ`7~ zS>ASJ*YC1WUUNVUNUIZzb~lR7t?m?&vzY>JyrQk2KSK5OUuAZw9?;Oq91F<_ZbkaB zo4f~KxKseMraM6OorQ4cnJhjYq=Gpn22$CnVBw}o;wtl8h&s7X%-OIDz8MI(a*#Vp zysijjnGc0$%}a}dyh(B*k^|QFqKs~fg@*H6gtVBoP;yxY4yZ`b&Z`E(pgZKpOoO_g ztEoB9gM5cAgf5j8khZXx-bu7@?oB;B@YfG;tE!^30b%eU!H}QqZlLDPzMLiXBN6MZ z`OrNn0`aMXwoZ${S-Br!=~*o-slFhz&Z&amEsH>Io|OHuy&6uQ*2HHXT~N8G4@Xqh z&=t=RHeYPPPbzbudiv@z^#~0%4(I_TK4-}HQZGzfqh4k)0{?1tyd6|>^smb3qgOEU|x0kFx~5_#|>Jy;Kvws@_5wZcj&=&%6R;Y znx&EWx!+EJtF;Wfnn``%h&6)Sr}xxmZ_8TOJh117PCQaGgrlEo;QG}*Y;5Pn)fO($ zoR|QYmMh~}{jF5eG+Ru&whfF90G99E4%LBksXEz>()ey!z<@|-Q#eEtonL+jW*B1i zt|xxO-#KGRO$S97dvWtBO=0MnQW~qg6Uwt6fbj=S{vaQWQByQo=ZqX?9WjO_=nmQE zEzm*V8z;JU=iGL6%#}#jNPlVItiau5wMmP&Ms?6C`yBdlDwvbYtZ~vu3Fl7O0d|F5 z_*=0Swp}cO-ww@`=G&+2%IaT`rL|Xxs*(}=<#_O3-B4+E`#~yPaDyx+s8MNq1yrxA zEGxPHfevao?QXx(kQGcHV=F*OuzRK)B^%&hjSzB_+>~-Uz_8sl`h+Iq&KWZEJUnFh`f$WG{6-FtAJ}0&nF`EylX2^DG0pV7X!u8# z18SUc^rGIFaWn{j%=IL@9hIQ8*%hN#wEQ3Cy?b1Y+y6H_Iw&!uP$nso>8Lak&Gq?o zk|=F$eI0g0gmKu>7CSp4#m;Gn5JN=loZ7Mn(OjP;Lc}I z1v0}dQ_(!t96c|U!1e+MOzvC?2h-0&*+CPGDZ3-Y+;d_{W{K4FzFIIewn2%|mU1^8 z1<8OzN^K9FpxhwCv%x+LHnzaQHFrT8dxfIUPKQXXUMNj&3kGo^C0&>aAwRW2$D)z6 zuh|5B@3dn&{=8lu*8>ygn?MZS6ctJA>@Mx0;n&IJDTd}n6sc|K2Mxf5o;+zGumDOi~C?@;3GMV<#Z8vNHx zaQm(TqHmv}HT470zw$N}Ea^{|TNTXy{VK@%x`s4PHz67CfclSS!C_Q5%dDME72VH3 zv&&X!nDH9679y^&cEFf@XGPV(8ql273Tf>U#X@NST92^9xKZ=Tb5bxkb{Pdu>8~M? z?_0urUXlO10dRiv05m>y0h(r23wbsQ_MvAdtY6rR#odjjPy2nDm${Nfv_>%XSDm1b zyr-;Nyd83Ok3jGJHh4eLjt%;`H!FN@f=&7PTv^ha3>nRIGPV!qsiuRmM-Iq-*O6+| zCD8B72b=H1P;%>ukmqT`GFNYgq-PsIQ`%1~whm(zlhQ$=EaK?TWzovsfyrMy6!iT~ zF)_Xw4i@{M{@`b&Wbrf5Fn>7NIRf<^uL-JE{e;MZxkAn%JrxL^9=-&m-*tlMZ@sB@L>;I% z8u6{xtwxls@`aa^t%A3--I#Aa1W;38^DUAo$-yXgkm1gAj4tq z!Zj;{ymdDq;mI{9ot7q?F9b|1{sx{C7E$&!bBwy+f(;wJndaSXoxHG6jDKv-;?~;G zONR)S_1mwIc(y;vADpJLbDdFh;fU`3t^g*9-46@%Jeb=YZ+t%44d=~rM3;+w*>-s# zmfZf8!eYOJg%9nS=ZpaI{uYdyw-bef;xbUGIy1FvPpEZng816sNVCC)zo(H_ck{vQ zp+@ASxks%z&!E2dWsn#$l?AW1iXk!QL8j$6$=!jF5buj|mV2RjMqf<7i>P>TN@#dK z9g29(v?kvM8^$JpFIO3-IXI*A7vgRH0MhVIEV6Q=5ZS|4D01$^8uDGR_exV-bax0- zZ>fgZZG)KRdcDqK$~BN$Or@lY$06i(F|TtEg zc~Nq6v$CM0F;h(YUP!gtPm;lTx`I`!1z8tM)K5;M{oZ!0>Ck<__~>xvf3P3qWY6vY zmG^rMQim|6sF#U;s(WO;c?8AuE*8p^4DNHcdX7^D#3pn{{dbA*#ZJ*wl{t$xZ9jrlMt9sZZ2-&J_<`3u<4?%A!m@l`A9V{+&I__fePIdIwe~|r`!!Uvqdj}pu^WrY7)Es-S7^c4AdGG)rV8^< zWHUVnv_mga_4YULY$A7N{MJN!b`N0H?k;$}({QF?K|+PwCh}YHjUwJFSnT%?V2yns zMpOi_svep2=#Bzg13rVnFHu<0nLBuVdXNRL{mY-kL&3;%RD0((**rN4vHMQK+zG=` z6F8SHS6gD_y`y56TmS3u$Z>ZLb$$5hH^Yf+Bplvl2toM?mmNzo^yrg z)tgY7wSvm*Z-ZvHtq^y+3aW$qvVzo$P~h91>KE6-utR>_(Hz7|0}(YtE6F(UGwF*K zkYefsuz8&U&Y|6ys={0+|Iv#rdg6(*W{Z&iix(E)3wl0gFqW^s1$9S@Kz`;ueT#R( zX8~sT&D#{cppA?wVdBs z7zJT(-$Rkak$EQjP>w}s%$`_5X}@vhXTxEk{Ioy#>6cR4v{l z7G-}cKkZU7-*br$~5DC6{Iuv312=cFwN==$^D=48pVgQ z{HNgsQdMz+!beyfN2OoN6j*Mo3tSHBr>*wisC;fMdKRfm1ff!m0`!U@SG=*SdMI_kkwpOnmz;e8#^$+<5V8v2+xq?z8@Zn z-miH2mu*MXTmm7rbU9@DnvtyYMp3?_KQ-kH;m_5MlDav$kT|vk>eVZ$q-``LSuTP= z=N{zdvo8cpRbb&!0L8Rp)bQd9eM@hT zslx^`=@?5^_*%{q8j8U}_#K+gtrpFE6igl8o8n3`=-YgODKx=?zsnieJfRz|-t~gg z{dvmLmwZw`|3a06GQiwxAlv6{gNG7LvDv1EGOl!C1s|LtVW1z$y6qACoz_BPU<*L~ z4TyGjAxX8hSl(tXXi|O?#vZMq6h~(kX9DChqzy~p3aYew+r$FuM7b*;Q4UYLN*dCG zG_#!H4deL8UO%jQ=)GbQwOfq7$0S;_TCFiiRj3J1vPqJHyaJbk8f9OHx4?-qlmxw|gy$Kg0jCSh=b zrySL9f(1P7r{$Rij2*d;Hf}RU+w#8nY+DF3d|Mv)Q+X;5!d5l#kma zPUH7hTY8TiQoG@CUx7^v?2Br#JuC0!gvqf@5V))iT)LPtPsJ3d+IkJtMVrJlWgiSY zUJps9T_AGbT8h8x!K(M=Kw|2BP|y4-%5DxOMU+mAymVhsFJC4!-Qj!kjYuJ%YoICP zDa{2>92<~LRbD1c67R-hxKlCvz;YR6Do$r=)UT z;nzID`)~Y^^M2UPdO z73CS;EbYMxj)lCY#Gt$6HbzI58@5xzxp>N69|U3Ff~e))Zj#rK0ol2fg`6L}|N_o$EvFi_9nZ+zW_Gaftkfk|6;KP}eF}NpF zpZg9R)4eHmM+Hf|&x#Fuo56G2KC-^EpQqoAA@BWzShddrs{T~LLr4P9%qf8P@kga< z{0XJ|2p^{D900L5u2GV87)o2DY-5u#KH)f6!t6w?ZZ)T;sf@}%rUmYm6gO0 zB=H2au}{=sX~Nwg7L^dUpa(vC)rplZ4W?AZ4G0`72ic;{f+8dvPWO>w?YA?O<#mq= zf_Ff;-&K%&oXPhRX*BPnoW1F~84NF7$-gX|;x0N<XQeP+#RaCIbC4(LDsCntdjJV zreOTJJEpm;r6lEBp|M>%mez@xl)9k}Smfa*9*A_|~QwAxzj`T*li;~KKI$f}5-d6iaRd*Nm zI1R(R3tT^Q&XiL7+yt{zez?mam}w%Ju66Zen(@$?RTi02L+MgTGI>k2Zr=#HwbIhb zgRnT|3)G*;0~>iOT)x87AU(FjNK-E+bu`0KD@`zN9_JlF6>t}leyX5+hr>|!*%VDLdN8NEpFmTwiu?nrXw4E=c7ELmbgW+m#e;IW z%hMS>FL+blx09fijsx!>yRoJnHw4Xy0A^g04eO5rw%BqUtHd82@kp4%rs%zG21z(@E6s5J^(UA__8j56&$fA{ev8( z=hSi3eosFpAIvewiJ{E#u?ZbK&$-1pP9&EPVg`u}(_RJ&hOD9FyVx65cNQrlH}m;- z&SUXVp)HI0>p5uTT?D;v2R4n@yEc>Rs4`&%wXPX}mdX2R+F}XH3hae(HK8oFvYgki zlW28F9@VUS3Mn_NK(W%A85Dz9(eo@wi?^YY!}U<%nGN~e6}r855SE|VNC~g|pn*Gm zlh(8fDGq;7Ld_#kb8K-9=LKbsdnsf5GSXJv5hFK^p}d7t$)Tq+CT*BPwr>#Y%r8RR zn3v?YXBntD?@_&IgOJ(R3nYIAf@sCsQi@vSykU)KsM|D8~@RZmvNwyef{Ii+-eL2B>KAb;pXszvUr{un8)t{a}olwPoa(-3`v4>WFp|Bk-jH(Q+#1 zqHK4Ij{V!AY*s%=j5JW5M`yDBw4E$==TZLNt5kQQ28xGVqvoG)LDbCQm^C{T-yANb zQ0G95ZPLTAJYSaf>K2q8vSQXBhA@Yoy_k1$PiEG28mvg^&*Delg-A1vkm>738H;UD zR{k$BQWYo^uImXQwXIY;Y#(^WK7(0%bIEgolF0^c0pEL}Xt6enkBxUav$oz?8E8ih zef>FB=p*FjiqNzvnaT@IS>{j4ps+Gwu`|<1XVM$fN_vQK%JY=qYKhf?C9cV^W5w^^ zlJvy^(C=RXsdibg>*+wIdeo@&&lMPr@#K!q23=CjOrB=z%UW#r!-9m~EV`Wo^ZxV< zT7C+nmtnk*ij66GyObwJ)`9G=Z$e4eB8qLi2i4r^Yr%QOjN^IG_{EjQ#gAZqJpDvp zu~Jlz$R&N8CzMLAg1XHxl+tiWFtG$p-Y)TEs1=KO|5`{n+l|@Gsv!N5N@b+zN0Ipx z#cSK0`I`2pa?*XCVEtk*#GQ*^olLlc;)M)l@?nB%R6FJ}6;QF>h~=oev1=2CU{dT3 zsJ>At+9#JnbfqQU{?`bmNqwZOZ#@l)U#3IroBgCVo(G1V#|87nHatBg3=OvYIejIE ztRgMck8@-SyZI&23CE#%%vwL2(&2O*(5 z-v@5It_%EXgfj~GK4i-#Xv*I~PG$9Ad%zFF9uH5L5jKXwx!E-eQSGb;}^F(_kvR{XLj3N8B9N z3m~$aREmcJ z#cw8kPK0p3f+reJi-WXhp-?cP2L^hr5G3P!2-PO};=V*LRz4vFrAInqnae;{dEga= zJm+ig%^=eLwg}XGPwx4!J4_qm%ATZ#u&5LK`Ss9(s&~51X#a4g8vG85j0a$Wq&-?+ ziGbY61ZKya=Ug0sNesB;@X9Z)zOGbCcWA`4m~heihMJ0=Y$4g_0@2WY3pqAiB;&hU${Fg8@x6YhjNujZsOM0O za$w|d zZ;O~%+lEPnBZ6M*Bj%X}qx$4yn0-aUbztYkMeF%I@bhCiHJTa~f z=YT$+qS_oe#C-Q!P_YD1OpYeW&TB$;Ydc78R|V-My-?*Aplj&c5gq*7qTNt`CL8h& zB6U#^y_@qa9~|h@9w&A?V+hwP^J@>gDMY;R!j~E=c5!bvX6x(3#)j2FuICMEDmf35 zLBUM9uRGKKs5<5L&6>%bRg`1%7u_B#!>HI6&VPR8we$$I|9X^)KlMY++BnWB1+uYM zTv?deI%-&W)c9CxoP^rI7kZPkgeW zGd_>9$4=yes-0U)0@roN+o7)LkkS|L|JjL|O5GZ zbuAlt8j8_RqPO%5`Bh&cv+c7XwcCEOJ=d4Xcydl)O#)r~A&@oA&*b|#Q)WBBoMoT9 z1!GrxGW)qbFw9j+8*?I<;l^CSG|7(D4C7eO)!QVA?IT`}cW3h3zYE4)Pr;N3TUNr? zpE?=mXODjd&AeXZB=x{5p1QVTt{+nb9pZJ&NQmjXR*W2KC6*W@xS(BEtW699&0(Z{ zb-t|bY7cZvXpd>m9w6^E3ml4uV(V01$8-*b(vM~0Sd(3(+S#c@F+0ZJuWl=)Eo~6( z+ge~uj2V@V+6^rOk^B#RAb;O>XzOalrewHccJWAhc4P!I#QrGQW-0OfP%F$D#VE4x z2O;SicUV$9w?IuxL@@9D%JHG9YdHR7m?Jn7MZe zWxe0?3Tw9*o)J!=+C8V#LAta8v4o?pDW+lh0$v*21 zDe@9%(#nBYI=v@Y_X)<9rp}C@j$(Et~pQ@=U zX$4rV^GENmeVJ-UJJGq(1NBol)^}tZD87cVlOH;=n8v%J&s8bY@9Ds@P|6A&)nIk9 zE6R=?5yr0PT$D_2)je()|T|JO!x8>olKsG(T5_828SKPYG{!|B`f z5m>(M2<7F@fRZeMlCgq}+xzi7UJ*30L*hHG%LdNhY~`B0y5vxn@Jk!E3 zp@r(1*37YBG}-sEWY+UekbYehEvoIn{O?aBp8-x7KD-1por_CKf3&27_qK9vcPgy7 zY{8OwI~G3PlZTyXD&Xes9p_HuKB<RQ=Q{MyBKo)t8pR+&O)japYZ6 z?DL@Num;F~Yr^ryJR$PwC$WJ$Qq@m7Gkb9}SaewhWDGeQzdbAEbGK?(4TP+2qVS*qiwjRb_v}Ub#o=l(UAzJ>%Yvy3BplC8tj^~O9 zv-M#Re%O^cJmLD3o|bfdWG?7$oD)|z_Q&h5pHb2ot)LBZ$L9MTuzr#w%WPRh6`Qr$AU;Y`Q@y&90E+YRoFeb!B?dpu29e6&xpKfj6%yR8231@@f89*EtaD z4t#}!zBOdHd;lujb_D5;-vq--TS$A5A*5`TLh19Km{Jyin`gVB%f~J(Y=00KP3eiB zUfQDBtqKa``i8)$)1=sYi_F9oa45iG5D zve0xi0Fs@WL9OYHP4+=dbMXndD14abzP%9qau^mA>=e?BcL}Q3WBddD^ky;k+eC%+ z7qOhz3X&NwVON=es>vsyiDTn>(LzWXa8hjcy9}EABAx74FDh*}0aDmlNSN#mMuEO; z;dOuJH`SQsTW+V?#jl|}`93%~@cY};LjI-A5cuXPbe+_mMTK@o$AH`9XxIkYQ(r`> z_3zXa{1w734#I=uLNLX|1rpEHg8k(nHok{H`h~TU(VL#APJJRQ3gD?`;~v8L@7=L% zW^eS{eTZW?yas*qr?M`ihI0)XSbw|&Gdd8$a@UkmAgrL4U(&cXqlj9bheKt-Ua;-n zgOw}{V9{Us7!URkliaonfj5m&?#cNq>#t(-jE<~fwi1i_KA_s?!}ktS=P2auId z3WuQ-^W}^X$9-LjqDNx%x)3n?xe%&GG|}AioJYJg0yW?Zn#nR{(qmpP`3++!BSIh$ zKT+M`E%Yp;Ey~j}1;y9}WcK|ckW?26)>nH%(YtHZz*86e|GGt*pCnkC6~dgXvM9vP z3{&MMXohYOxzPX+ zWpbThIVlQDbkSCaA)!?VTL)S)@8%2g6)>L(Un3Z$C4Qo7bPh)ud{kMbsOme^jw5~DAWnc2lZRFQc-RwYkrtP*8O{e+_?r^7Mo)JFXc39n>ouJ>cb?r zel4k8GLss_d$gIlqqdzlL=OvRwRRIp^1fWx`&%b;`gt8Z+u9M|4eY@pKRgk9xaPO% z+XreolrLCZa=_f`E?9l}vyfP9gD;<(qduSSCr*rk47+U*on@f>^;anN-d>XL*+Y#% zfmpihKB+U;P>~JiJI#2#dgBR2&9LUWLJKx!b{91Edj&Hr9kBF-8_2!?5WGhp=9+-f z%GUQAC{Qk7j!ce5ce*f(JATZ6O(Q96)I2=C5<;eMUCQk1%4z-Wn8xL$nBDFbHC+v+ zaK{5ssJIUHbBtuxBnkUD&)WZSj$<6|YB!V``7LP-ZSc>G%HRZIv{ z#DvxLZHrHQJ7M)aTNc};0a7+sKvt(t7-~6yr;W}j(PWnksx3d2&~!ZnhMW);5mEky z$y{^J_jV4OIae^o8LYN2jT#$;j8KXgQqJ&nL>;+A&4Altjs}}^rApausKI5 zeVQQV^H;(AJ1m4MmB5;)Fs$|rV5MQV#n3Ymtg2@wX@7VtCe4(9_Vy8Rbx~(j zXI~-fZ|%^k7BMY+CON(z%o;w-gq3>kwOe|RB7Tsv?3Y&TGT&pBRt=oP@@ner|pE|du3pt+Wz%xpBnEN?I7f87kiO}#MX^$Us~(hj8)xRz-8 zS5f;RoeWF)2{iU=gp$?Y(dD4wSXyuc4&KiO3D<%zd^w0!F4#_*9o^ZQ06T1cB(RpP z3E&nWlC)c$cpiYKv;;s}K2LPZx5C^f%V^m1HZ1M1nUHyYBot}RQuTWsmGFe3VYl6w z!F4}HnpTS$$2d+^&7iPg1Zyo!hp6mJ(8AO4RF>~V|M(%GU6lwHySH;67WYwg4p2IVVK5|Xis>{@1G!_%JH7~0er0Q9tq|9LLh!KKT#@G01fAXlj>t2 zVvr20ZVo|1emFJUb;l&g73;^>6Pf)Eab4#@!O#7fY}*q-6ZAlkZmAX}1#guF*G5yo zqRuSs5-{m&CMFo3P(f51pdB4q(ju;v$vy<7CS#~@J=cN0nOxEZn%$weY$&x*wdQ2EPZMsgGH@*XxOA(mY<2l%Qc4yjVXK>84gEWgxf`ySU zOZuMc3RmY-=Af~pkZVf}JQPb1%F;0fo{p>-S+#<;1*NR;mZBTx22`Iii7f3ObNoNcP z&A}MR?sXdUR}To$OHDv~kObLjOZeizHTR#?Bt2(>pBuqUOURPC><7J} zgBV@?7#4CI-YJ2fSkQ4283N2vzUwS`*PNhd4?OXD$uiRW?o}p5gpqdGLgC=RVQ4MP z0;x}#cx_-m)PHX!N&?6FU*-wwQ4>0&e*YMN{xE}7sh_jKBKVA2&MXwTtLmOajn z_$jQQPEQx@rUbEU8VRcB&BFOz0a*I(in#CdK-Awa6Ys|iWS!nh89dj(cx@l{srxVt z?|Oxf6oj*;-Wx>O>Hesl_n!1V!?|D3o~ec}S32Gh>3aND7(4q8RV|KSi21R1 zn*EudJ7~fbMJ_x&z7Yb=?^E`47pz%oiBI-;VA-125Wc}0r5oB&V7JF0Im!7?v9m7C z+6-#VbD%EyID~m~T*YjT5c%;XS*m&Z&60PJJg1K%&pvGX-2I}&QiipCM4;(S-G&;3;j3!36h8)p}e*$ z_v&4xsN5PDdBy>+1t_o*zJou$4M4s1HYoGdgLVA^Fn-sTIUn|5&~pS<4t0c+?YYl< zg9mEAUn@3UsTCAw+;p1cyX5G03<`g+f$D!X5gmR?sf+ePw%ucz&Arr`es6UCdpwzs zraO}u-xnf(_JgQtEfl|K3)M!xq88(obo^u(^RLr!Ut5jfV#@awBTPxLCsQZOz9?G! z#S>o7ujC%~FC-P-2u)ve#6zxaQN7}!7&w6^+n=%{Q#sdPzjsEdT^KdJ-6bYlR6|_Y zHYymjMvUq8lx{bN;N>7&R)3tQW?$~b6Z3=4H(Mp9Uj)&^T%{Y>7t1FE7Qk=!#E%15}fJ+9uUZuwPoOGhlY zQ!R#k){@a#Ka5t(prz^-sRA-dWzqpL?^p(SRs^v~nN-Z)G?F4GjpQE4gW$N(lkfk$M8%zUq`!Jh zFo<4a#Qh|n{f)xjcVdcy;Yzt*4EcWRkImsbXn~hAK3~!gliWr_@7u<#q|}W? z2D%H1AIn7r*B%BcMPVF2$#Ph0Cq7?Z2J+epigOm+3)~wmdKqK&^8)z75FZ@_mgyS> zx$DnUOYA&|8ITImb#_!T+=4m9KY;4ix8!|l1g`nOxj(O=sJXsY2zM%l)rWcV{kDr# zHOQMajOqg4Mh(Tp1{;jNWyGdTG{Rj&16Ub(6aifGmF@~ozjbFCc@{}R zCkxWd^P*za5W)IS3nuq9hU_*TxUfSQ8-L6Zv-{Yyed|WBsPu2tzDQtMR?k5@HwaS> z{z01eA%b6{FLPR)Nm3srr8?MPYt~Tod^wy9?n?y2cm7m;?q|@vm?27cw}X>aUMxTU z4A(`TA3V!u2Q{smPKi)O?o~Zl{8+9#zg{4!cXKXwX}z#|ge#i8?~jJau4477z4Wft zi^)>lgl_>nYhauQs9W4fKV}s7G3And$1}8RX?IK=S_`AEw#8zT8d{j%1|zM4#OpyF znA*BQl$|;&=1+bIhRexR_BaPh-29+W6M@+i#*mY@4+ie<#+vu_XGsITh>4oLVE8&4 zuD^UplN3IvdLU)@pK@$^59!v2TA-)#FI<->BkAa~bdq!Gb#Yl#oy|RDxpuVZDX_p_ zzJdSAfoOPjj*QQ{vYIvFtTf-BWXIB>`QdpuQWU`&o(*S(Q~QzqwnoT!sWq>vb?(@&KB@75rB3qMFihmY@0(n*6=_d|5(&&qex588cg^ zKzZk*l(VP~N^8bZS)XBeQt885e&SqQUN>sI%X3UPMrJAZU^$aNQM9cY)=^wDiPl<63omlRdrxe}ccM5GXLzV3nWo^3y5K~gb>xdWNu(B-% zKKmfJ88oEs)tBH}1nMS*VEH2r`E^`L(6s~_)*a?Lyl-IGbASxj+*#UnceI#pjCs4f zAvZ9aa_wwbie@F4eY*#Te(S?#TU@6=fBa*&plXDEv8YSd?9DIsti=;i zdZuz&}MOV8yBXZs}VeHY?z|12qM>(K-ke9yr$tvrDo50_QoX`G&&fql0q=A z*oj(?^g{O&o)}+6pt(OssR_KxwPYJfFW51?|4pSdu^TJ-+@CqQ--NWi&p@3!4n~KE zvzo~kOw!}45`NvnwQ+aEQ9%+^&C`p;>JCy$W;e|iuaXV|a=#~ey>eGdnxA0nqg zq0H0Knk_AruvnQtTkU7T0)KD-SjqhXjYpvQr`z-;r4!04Y#?DpJSFUsQw5e#<`O`) zaSj2nPl0|n>7+Nu(zJRWwQTJ9&Ob4ExZXt~lEX@4rtkF=g-B0PnR8pF&^h@VN;?V9 zi|v_o(jg(!ZWwEl&m_q`B~yRZG*rooL_4|Fdo{> zJ$iTDSVh<-Na<#vjj6t@c)%v;18ukBdZ@ZwnwT&)D1~Y4I9R$XN!#Ex1%MVJJ zcWpB?zA<5GVfkXYlQSlEYyq|M8AXQw$T8C$6nH*YR1e9$4#Sh2Z6=?VM%$>y-5>sOW7Ln zg0dgqrR4l#YTEOPin=+my6SM0e11qFr#XkXt2bNN!G+hPUqy4CjS~5&JGD&a!~ zYMr_V!Uhh)(zY*2I(#!r$E!@ zI3#!;rh`p+q!^u|D^HX|aYYNIWY2^(%feXG1u2WZGmuI>)ZjK{4`qxAW65=EsB*#@ zXo*?@lJk=Udya!ucC>@bb8T_#ZIN24!r_nT;pleYFlBGL4AEolV8y3iSeP=Fk{tPa zR=fbMmy1}q{s-{m-k(oP2Q&Y+B4m6H$3MQAvg;Pz+4^et?n1$Dm>J8Q_yrUmDj`!MV7APgO=}EcX~|ch(uC`M_5-@C>dLn7w`Nh@+hI8O zZaCg>WAD0zu#BWM(3m$0#Sd~w^|(Q4x%Mno{wo|5mj()ZP6pwOZobTO!a^8yrzhKf zE(}wq&4<;GpOABNTbA2nFO~CC-TzLJjN~zn|D7q4{}oyHpDB_T|0BQd|11CeBk79@ z$t2_df2S`_N92n(@tN5m)56Z?7GCIJTDEP}M1L3m+p6prKPoj-VC4QuOc+11T!@NH26p8c4{9va1yzq2<>2+d-H zzmA{K z2b%bA_M+cE(8PbU7YF?VP5d`|aqvIT#DB9F`+NTbZTvU8vH$-8>r4Ot0~VP6|Ll_*kZpjpzaN%y%nA<5TXI!JHxx4wccTUpS8Zf*ILi-`r|(5T-QEl@BP`sb?&pbrY46YusF=uFNAYU zmBZnW@>;|h%;9jBL`6pXL~#V7mWBIq7=fQFBD@t}uPCqYvx4DXkzR|JaC-Q8kI?13 z;~l=={x6Bjeh@w4J>cIH74TMhPqO-tM1y#%9Klep#eSS|1BAL9A>P5>J|xha!>{-K zLOIeeB$_uVpvO@jAQ@0fn0MmsbGTlG|m>Cl06*WnpcP6+baJe5xmUlKnmopI5 z17n09hbO>~G3dM9-VR(08Qw)&yVruKxfs0Cwj~Fc;~?Zb8Q<;UqF=@#Qn^DJLV~w} zRR1OTdPp}+XJ4MWfFd?KVO8B}l%8@IShGa9-I)Th zUHY)4`wz5k^}+1$lUU%|z#a%Zjef}|;ayKQT09YhTONBc^4rP(I$VNvCkggYm7C|>gwATil1Z3b; zOg7H7u}0;4YFJ>dgKO6Sb{&WUiJGfyXv8aG;3Pvo{h^81U%V$dcj7QLH=ru!h&T=n zn9uIIyPcY~tAdiS1=&i4Kvg6bTHoa0vZJAN?8C3Lx77`XA4#CH(J8pZd;)0@Z=(E1 z=3`pxALL$e4{Otq17k0`vktxt8r^!x1WIK<;T|Wrn$}A#9&mBxrqYTt1@*L$e-p{| z`-7;xoCogLT+vA&97L^BK{8SSrOq1@wz!roayZBJH=B&FZ_bA2c2ZC#Wsa}H!L3c73?0MnvL5Q*gf=wNy8Op< zcSjj%oid;H=^SI4t$Ueu$uT&x%@SQ=%y1Uhm*#&Cftz_*p!38H1*#&WJ=sc)pX*|P7$uvE zg>avD1=;AZkVtxPG1p-~vm_`BRW@uUvo{_kL$=%_ZW^v2w&DbBbKuf0%7{_VeL;2MOFFIuv$2D`DbKwK5THdq_`%B^1{e6R>m#j&M2%YxCAi7elp3I(?Qg z3~YCNq@{+1v^qu}FSKibYLNt90}V`)I7ThRc9X@5YGmUIB|MO9PvWn+;|kLvO8mBy znTnbuE@L%msdmM{Z8mWKxd$H3AB|%))ZxJAV0e}50_9_Ok|je|QGt_^aCIc$Yw7o- zsM#KM?zFN(r_&*2oGWASU;=DxKFWlsdBrsW4t%4`^7om*yTnl!? zmo0g;m;X6w+E+{~%qrM<@28O^lU5OD{wt(kZ8WjU98XS8jv}7}!og#uA$%K^35`kF zIO}L89`+uA8s1|eX!8d~Np%E@E=z)w+A`31wugos8pX;>ye8WPhT@xYf1I-ACEahZ zo%rg=!?~1vMk3}mEgSQNjLZo{%gpoU?G#nK>}}ZWTEjJ|9a?Pk|>pDjC}Z9Zb5+ z4{<9Gk(gup(5NkoY#*ySO9j0}MbYy_HY}*RO3Mo+@gDz9rgrmN!VFQwi#>L@S*e+tn!aEo zAKSsCyTN!^rHAau=5QYpH`=%75bfBuo~~VZh*@`91G1J=x?=|~XCHY+?cLmI-#aNV zvXDZjkSis6-GvW`WY-JQ z>swFLJ9m?`ZZ}$^U=6NfshD^p2@Eb=Ae{$ZlH_|*P#m6(a;C571fNzK{wfx2w`HTO zK_-^034@^xKHyw3oG8jULO`tz@d;#Hmjre9L1!-q;zvo!-Jpd_Qem6!K+ zour?Zrr;ebFYu0dM!$WWhMH$qFd>ne_@XKU0{1DRyUh%u+N{ic9&HEtGfy)X+-GFQ zo-~k|beOhJZ>NLQ(m=wumc3~<9b06%Ad^%`0tQtuvkK)vy&xT|r*q-;-54+_O$4(s zqfuJWnCR@x!I3Ghz_rSP;a%ehL8Fonr z(cmwgbX8Y2*cCrD4d%Y2?FvUo!qC|;rEUpE6jl(yNuyA%SQEzxNHO-Ew)D7oKAC@E zD2xCTe0Y`*Z(K-*c%{$e!U8)`w2$OAt;>W@hH>~JY$1L)l?L-Vlc<`DK53KnV2KDIkO66!G5GwWLq*4SnR&Nc0#XoOdA} ziw=c@fKDXMu8YSKxmZlN^rUi2(FpXK;SaZ73Zv_nRA4v{h)}USMl=|aGl#XX zos>w1swERuKYLgZ@{Gzzoi+_KU|_a^Gw3{La9+td+L9PSQVqxAmNguzu~r1z%9f&m z$oa_!T++a$gP*y6IuaL4ts#?>J<)dZG&rO_2evN0&wLTqf;Y_BsyXKCnaRb5a4B37 zM2*ey$zeZCn(zlXUyz75PrV|8RIAATxL$#me5`yN%ut0#T^O$(7C>dt*|r3i4O_6r#%%~yb`dPWB~s}BTxxQgK9Y; zR5|Jean-q`HoK8le=Mg0GTs;~wwI}m0j7PJ6zb|*quH4PV*OwYl$TFMX_@=9nVA3& zzNuqdfEG=b4<>FUFX+X$wkW*BvMTWU2dXT1oLJpEM6Q|ovPXS3lDh3>Oy1{M>g9HW zz6#$)*GZbAO!lo2h>5N*Lu(SS4}gB1!6w!c+%-h+R3BZP#?9Z*uKW zS$R3^UALVncrY6kdVFxWUKXo&+nzkK6re0v;F&x<5N{43q1s8H`&I|_4ILpTY$sKb zosTy?0PdYng}m$2;C77$zDZ0&2lfMTJp^R_rwoS7vW2FS<4hlaEm_<>54&wydU>ll zR&Q@)=%5(fl2A?)1^uDFbvRo+ql1Kams7#l9aLoR22xb32u_CAs0j0tu459oK8C?K zYnB}B-Xnni8fWRqs*$N5HRdBSL5AwVz+-OZklOdbKr>1%16FCX=cHX`q?%;7HP3jp2jWu!}32W%48GSxE**tlT= zxa(UCxK!PsL~9ye9TEl8*Q_GLsvN5-WvXeaXEs(n;)8o3_84~e1f45qO&1++qc)v4 ziT+3_=voCGtE%7F9tLU`H04`U@%#mh!XaX8R=?O zB-?Cg*Riqi`Dzkm$)rJY*A=Rt=85&qk4aL2269GNlW|Ke@tXA&`m)Cfldi0U(;x28 zh0849{<9pK_1X~mFLcxRMX#vQ_+gmnx)R23N+uW7of#!#52(?6%}UyFh&KX=#P-t~ z=R_!dIUo0*JWk&%?V_KY?lVr25!fD)3Lc>q)b_Lqo1$lnbCg<`{q|PKHWrZs;-9E< zK?bbfphd=5UFFJkts_ag-r(>h8K33FaWzC^v3VFz-@-Tun3#f*pC6H^r4va)Oc0j) zKVjahPlE;B8K^Bj64#ne!`MsPX?fpV(7Bn7lg8TPiP}HNkijyiLF$oV>~;mI04#Lo1@Vf z4N}k&3yUtM!^Y^1q+wGw7-sCIy5iTUM;|}Fk#1*NuPv+6j-QDij!&SswACRfAO@!% zbA<%r#k}ubO=92Xz=jj!$WmrGwq5EZ(^}ms?Z0qP!FxDK$V)+s(LE$QJ%P;I-Aq(l zU3l@TfNDa%co)kVX0hM*{JKzY5{3o(kf$1N*{AxOxGh*uMP_Hhhir4;U+V%DP(%G9 z0s5`+_t5x7$K0)1G#iBSrpw7bU~ z`;|A+#-2-6dge~h%8|ha@dW&$_MGOXbkf4KVbtxiI!)n^!QJDFNaSQ)c4C$Zv`k-1 zu6@)3^>T6K_YH#ct6%coxkKQ{&?vYhU_meX>Oxas3K`=&5ksBUkc+};z%QAClAWh$ ziAXf<4i8}q)^u`5MQ)@zho91>M=nsgFa<%`gHMl6gyriebdBO%$f ziahL?jxvM!P*S0U%QsDk1g`3)fo}8AD`zX?u96N{Kg>e4@;UUy6L&1FosY(U6{duun?yeS7;A}ATSRd0`3~wT zZAyCgs=$32Rg$BAn2y?MkB-aD>HNeCB=?vvF}dW1$FFcSwcHaeyN{;cKTmZ5(i}#1MGY zPiS&*7znSlVQS)D(9Nrr$)J7viOM7fuN-y*YgPd0ghAxJ>toWH8V@-wF;uNL3D4N3 z(zPnJH2i)R9?d;LWYr##dIM9CU8IcrWw+3{f(5Ws*aWLYoM1td5Lnc|AR<<`s<`1* z4tPv>};c=?kPB)iNnBe=SV$0!77z+A-nj_(~+=&F5GvAzS7~% zD_mz1WZY3d$Oh{|&y&Jciy(U40#qXcP{E7$pu}&CVwEj;E1AICT|?lYY!+eVJK zSYX`ncmA!`n=#I4%rR%bqZjtS&dSV;XdCiAJ-8P!uS5RJCXn7jyE` ziAtg~nG*BdRIXD25AJouJ(!7g8&%+S(;X7Zn!>Y}>Y(S*#XdQifl_a*a94LTu^-Y# zWn6}%O^zA0%m_oy<-@d|-x-!0DnW!05|2Iep#8KPTV10{?t>ZXIR@cPQ)lwN)R8r+ zT!>TKV!i8M>>EIml`cw|iym5)C;rm3z!ZRRtb35tm+C&O3 ziP3AHZ_p3Jx%6U4AbRIT!&lB(DqlVoXkaIC9ur11KDbghUOtcsyG6@vYRH!xQt&B& zAycY{?~#=5lVxhef@8-tf7-=#A?isQ7ivr)0d40|Wp zk~4Xo;d9`+&J5i6HGubxPZ(=zOt9^S9Ldq02!)5e z*?m(}*d31(AaQIw`n_Gnidu-n8?pkXPS=IAQvR5ot%lweTqr0LguS;y;q>x_Wcw;b zuv3qKmD7u>hE3C_PvW(pZf7h}8C^-u$!Ju)pTn9+Xrr`{4u-#S0#oCIRSm0Gv*X6? zVi^5p@I=9a^jsVa723WinYx-(b{rrFRfjTTHS@@dKh9F2OKr@UZcTj60)Em}qW4Q@ zg0yEYT{w+FQ`MEYDyWu)0asV5sQVpa+%*V1I5ebFP^KF?Xx0 zwh6E_E=Cd$EJ%ivlgj84&50;i8&4FR#i&(@5$td?#(_$R-G8e9__O--s|4`#yZ^=S z^?<+2&u^(_>2o~)Kd)sk_)*Jt{cpMQ&uSKr-S@$(X8kL#78~SB7vy;Z=ybEDmPU{t z_lcO#*Ta2f1R5H5qh9_l*r?==*QGf?N=wkuU=`ZSslh?7ep1gz(5g;{_(WwudhlUL zkxU|9%I(BAAq{9a1Ib^cF^*TK4xgG2^%KmXV_;jDGwX?v(^E8QoK@k*N z2%?^MF}#|65C*bD7EjUtW{vt6v_G?i;7_D^Ma$n%FCJhn==rmwX$i-N*HOZ2o!Y^h z{r7Dm9=uf^(f?f+#h)7{ezau$sXyvhZ=e z|DA}FeuIdA5k-UFAmU#{(eO8j_!m($`VAufMHG#HgNT0-MU&qk;$K8@@^29Fuc64{ F{15PjC!qiU diff --git a/src/neural_networks/NN_paraboloid.bson b/src/neural_networks/NN_paraboloid.bson deleted file mode 100644 index 313a893a8fd971bab1168a617a533350d5a670a7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4668 zcmcB!Vqjp-%}+_qVK~Iiz`#_Jn9jh?z`#&kQdF8;!oXZoS&+(L!wi+-1j?i&mL!5j zSqc)15_5|gelh`#Fktuyq`=y-Dq06o^Z;l8Zbfr}vOtqO@hV~k$}+IzCFZ6w$b%Fb zFt7nBKfj#JBnBo!Fh{v4HK!CP$ZW`<3{nE}6bF!zl$e}dl$ypM1R_Ar1OsLxJgT^0 zs-RZN0r@~HL5^_CDXoCn=#rXOoC*{J+V}vZ0%RlBa2N(&G#d6^#Efg_{GwD?T!)kv z{uWLP8*;kTLLx*BJT{5KMc!57+LTEP{VLe zRghpf#0U0|+*MMzF&fVdMR%Aj&ZZeP4i`o6*&JM6Y7?Agr#EZOp5 z?!rh>R4Zj6Q3noupk=_YI6uNu@i|B;J^%}PI;CP@B@G6^R18bd(9~f75)a@rviEv@ zY@g(UP5To1z4z_Cd3JB^Lt*=-xcmF`6Xo}rHahNKJ4?WRxyp@wPX8Uisd^$%A3akw z%nML=^MKs_Z_`~{5PkP~jNQPbXd{NjBRoYff~4p 10, relu), Dense(10 => 30, relu), - #Dense(30 => 30, relu), + Dense(30 => 30, relu), Dense(30 => 20, relu), Dense(20 => 5, relu), Dense(5 => 1) diff --git a/src/neural_networks/NN_test_serra_parallel.jl b/src/neural_networks/NN_test_serra_parallel.jl deleted file mode 100644 index e5c8bbc..0000000 --- a/src/neural_networks/NN_test_serra_parallel.jl +++ /dev/null @@ -1,31 +0,0 @@ -using Distributed - -addprocs(4) - -@everywhere include("bound_tightening_serra_parallel.jl") -@everywhere ENV = [Gurobi.Env() for i in 1:nprocs()]; - -BSON.@load string(@__DIR__)*"/NN_paraboloid.bson" model -BSON.@load string(@__DIR__)*"/NN_medium.bson" model -BSON.@load string(@__DIR__)*"/NN_large.bson" model - -init_ub = [1.0, 1.0] -init_lb = [-1.0, -1.0] - -data = rand(Float32, (2, 1000)) .- 0.5f0; -x_train = data[:, 1:750]; -y_train = [sum(x_train[:, col].^2) for col in 1:750]; - -@time jump_model, bounds_x, bounds_s = NN_to_MIP(model, init_ub, init_lb; tighten_bounds=true); -jump_model -bounds_x -bounds_s -@time [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750]; -vec(model(x_train)) ≈ [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750] - -@time jump_model, bounds_x, bounds_s = NN_to_MIP(model, init_ub, init_lb; tighten_bounds=false); -jump_model -bounds_x -bounds_s -@time [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750]; -vec(model(x_train)) ≈ [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750] \ No newline at end of file diff --git a/src/neural_networks/bound_tightening_broken.jl b/src/neural_networks/bound_tightening_broken.jl deleted file mode 100644 index ae4699b..0000000 --- a/src/neural_networks/bound_tightening_broken.jl +++ /dev/null @@ -1,677 +0,0 @@ -using JuMP, Flux, Gurobi -using JuMP: Model -using Flux: params -using Distributed -using SharedArrays - -""" -bound_tightening(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false) - -A single-threaded implementation of optimal tightened constraint bounds L and U for for a trained DNN. -Using these bounds with the create_JuMP_model function reduces solution time for optimization problems. - -# Arguments -- `DNN::Chain`: A trained ReLU DNN. -- `init_U_bounds::Vector{Float32}`: Initial upper bounds on the node values of the DNN. -- `init_L_bounds::Vector{Float32}`: Initial lower bounds on the node values of the DNN. -- `verbose::Bool=false`: Controls Gurobi logs. -- `tl::Float64=1.0`: Controls the time limit for solvign the subproblems - -# Examples -```julia -L_bounds, U_bounds = bound_tightening(DNN, init_U_bounds, init_L_bounds, false, 1.0) -``` -""" -function bound_tightening(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false, tl::Float64=1.0) - - K = length(DNN) # NOTE! there are K+1 layers in the nn - - # store the DNN weights and biases - DNN_params = params(DNN) - W = [DNN_params[2*i-1] for i in 1:K] - b = [DNN_params[2*i] for i in 1:K] - - # stores the node count of layer k (starting at layer k=0) at index k+1 - input_node_count = length(DNN_params[1][1, :]) - node_count = [if k == 1 input_node_count else length(DNN_params[2*(k-1)]) end for k in 1:K+1] - - # store the current optimal bounds in the algorithm - curr_U_bounds = copy(init_U_bounds) - curr_L_bounds = copy(init_L_bounds) - - model = Model(optimizer_with_attributes(Gurobi.Optimizer, "OutputFlag" => (verbose ? 1 : 0), "TimeLimit" => tl)) - - # keeps track of the current node index starting from layer 1 (out of 0:K) - outer_index = node_count[1] + 1 - - # NOTE! below variables and constraints for all opt problems - @variable(model, x[k in 0:K, j in 1:node_count[k+1]] >= 0) - @variable(model, s[k in 1:K-1, j in 1:node_count[k+1]] >= 0) - @variable(model, z[k in 1:K-1, j in 1:node_count[k+1]], Bin) - @variable(model, U[k in 0:K, j in 1:node_count[k+1]]) - @variable(model, L[k in 0:K, j in 1:node_count[k+1]]) - - # fix values to all U[k,j] and L[k,j] from U_bounds and L_bounds - index = 1 - for k in 0:K - for j in 1:node_count[k+1] - fix(U[k, j], curr_U_bounds[index], force=true) - fix(L[k, j], curr_L_bounds[index], force=true) - index += 1 - end - end - - # input layer (layer 0) node bounds are given beforehand - for input_node in 1:node_count[1] - delete_lower_bound(x[0, input_node]) - @constraint(model, L[0, input_node] <= x[0, input_node]) - @constraint(model, x[0, input_node] <= U[0, input_node]) - end - - # deleting lower bound for output nodes - for output_node in 1:node_count[K+1] - delete_lower_bound(x[K, output_node]) - end - - # NOTE! below constraints depending on the layer - for k in 1:K - # we only want to build ALL of the constraints until the PREVIOUS layer, and then go node by node - # here we calculate ONLY the constraints until the PREVIOUS layer - for node_in in 1:node_count[k] - if k >= 2 - temp_sum = sum(W[k-1][node_in, j] * x[k-1-1, j] for j in 1:node_count[k-1]) - @constraint(model, x[k-1, node_in] <= U[k-1, node_in] * z[k-1, node_in]) - @constraint(model, s[k-1, node_in] <= -L[k-1, node_in] * (1 - z[k-1, node_in])) - if k <= K - 1 - @constraint(model, temp_sum + b[k-1][node_in] == x[k-1, node_in] - s[k-1, node_in]) - else # k == K - @constraint(model, temp_sum + b[k-1][node_in] == x[k-1, node_in]) - end - end - end - - # NOTE! below constraints depending on the node - for node in 1:node_count[k+1] - # here we calculate the specific constraints depending on the current node - temp_sum = sum(W[k][node, j] * x[k-1, j] for j in 1:node_count[k]) # NOTE! prev layer [k] - if k <= K - 1 - @constraint(model, node_con, temp_sum + b[k][node] == x[k, node] - s[k, node]) - @constraint(model, node_U, x[k, node] <= U[k, node] * z[k, node]) - @constraint(model, node_L, s[k, node] <= -L[k, node] * (1 - z[k, node])) - elseif k == K # == last value of k - @constraint(model, node_con, temp_sum + b[k][node] == x[k, node]) - @constraint(model, node_L, L[k, node] <= x[k, node]) - @constraint(model, node_U, x[k, node] <= U[k, node]) - end - - # NOTE! below objective function and optimizing the model depending on obj_function and layer - for obj_function in 1:2 - if obj_function == 1 && k <= K - 1 # Min, hidden layer - @objective(model, Min, x[k, node] - s[k, node]) - elseif obj_function == 2 && k <= K - 1 # Max, hidden layer - @objective(model, Max, x[k, node] - s[k, node]) - elseif obj_function == 1 && k == K # Min, last layer - @objective(model, Min, x[k, node]) - elseif obj_function == 2 && k == K # Max, last layer - @objective(model, Max, x[k, node]) - end - - solve_time = @elapsed optimize!(model) - solve_time = round(solve_time; sigdigits = 3) - # @assert termination_status(model) == OPTIMAL || termination_status(model) == TIME_LIMIT - # "Problem (layer $k (from 1:$K), node $node, $(obj_function == 1 ? "L" : "U")-bound) is infeasible." - optimal = objective_value(model) - println("Layer $k, node $node, $(obj_function == 1 ? "L" : "U")-bound: solve time $(solve_time)s, optimal value $(optimal)") - - # fix the model variable L or U corresponding to the current node to be the optimal value - if obj_function == 1 # Min - curr_L_bounds[outer_index] = optimal - fix(L[k, node], optimal) - elseif obj_function == 2 # Max - curr_U_bounds[outer_index] = optimal - fix(U[k, node], optimal) - end - end - outer_index += 1 - - # deleting and unregistering the constraints assigned to the current node - delete(model, node_con) - delete(model, node_L) - delete(model, node_U) - unregister(model, :node_con) - unregister(model, :node_L) - unregister(model, :node_U) - end - end - - println("Solving optimal constraint bounds single-threaded complete") - - return curr_U_bounds, curr_L_bounds -end - -""" -bound_tightening_threads(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false, tl::float64=1) - -A multi-threaded (using Threads) implementation of optimal tightened constraint bounds L and U for for a trained DNN. -Using these bounds with the create_JuMP_model function reduces solution time for optimization problems. - -# Arguments -- `DNN::Chain`: A trained ReLU DNN. -- `init_U_bounds::Vector{Float32}`: Initial upper bounds on the node values of the DNN. -- `init_L_bounds::Vector{Float32}`: Initial lower bounds on the node values of the DNN. -- `verbose::Bool=false`: Controls Gurobi logs. -- `tl::Float64=1.0`: Controls the time limit for solvign the subproblems - -# Examples -```julia -L_bounds_threads, U_bounds_threads = bound_tightening_threads(DNN, init_U_bounds, init_L_bounds, false, 1.0) -``` -""" -function bound_tightening_threads(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false, tl::Float64=1.0) - - K = length(DNN) # NOTE! there are K+1 layers in the nn - - # store the DNN weights and biases - DNN_params = params(DNN) - W = [DNN_params[2*i-1] for i in 1:K] - b = [DNN_params[2*i] for i in 1:K] - - # stores the node count of layer k (starting at layer k=0) at index k+1 - input_node_count = length(DNN_params[1][1, :]) - node_count = [if k == 1 input_node_count else length(DNN_params[2*(k-1)]) end for k in 1:K+1] - - # store the current optimal bounds in the algorithm - curr_U_bounds = copy(init_U_bounds) - curr_L_bounds = copy(init_L_bounds) - - lock = Threads.ReentrantLock() - - for k in 1:K - - Threads.@threads for node in 1:(2*node_count[k+1]) # loop over both obj functions - - ### below variables and constraints in all problems - - model = Model(optimizer_with_attributes(Gurobi.Optimizer, "OutputFlag" => (verbose ? 1 : 0), "TimeLimit" => tl)) - - # keeps track of the current node index starting from layer 1 (out of 0:K) - prev_layers_node_sum = 0 - for prev_layer in 0:k-1 - prev_layers_node_sum += node_count[prev_layer+1] - end - - # loops nodes twice: 1st time with obj function Min, 2nd time with Max - curr_node = node - obj_function = 1 - if node > node_count[k+1] - curr_node = node - node_count[k+1] - obj_function = 2 - end - curr_node_index = prev_layers_node_sum + curr_node - - # NOTE! below variables and constraints for all opt problems - @variable(model, x[k in 0:K, j in 1:node_count[k+1]] >= 0) - @variable(model, s[k in 1:K-1, j in 1:node_count[k+1]] >= 0) - @variable(model, z[k in 1:K-1, j in 1:node_count[k+1]], Bin) - @variable(model, U[k in 0:K, j in 1:node_count[k+1]]) - @variable(model, L[k in 0:K, j in 1:node_count[k+1]]) - - # fix values to all U[k,j] and L[k,j] from U_bounds and L_bounds - index = 1 - Threads.lock(lock) do - for k in 0:K - for j in 1:node_count[k+1] - fix(U[k, j], curr_U_bounds[index], force=true) - fix(L[k, j], curr_L_bounds[index], force=true) - index += 1 - end - end - end - - # input layer (layer 0) node bounds are given beforehand - for input_node in 1:node_count[1] - delete_lower_bound(x[0, input_node]) - @constraint(model, L[0, input_node] <= x[0, input_node]) - @constraint(model, x[0, input_node] <= U[0, input_node]) - end - - # deleting lower bound for output nodes - for output_node in 1:node_count[K+1] - delete_lower_bound(x[K, output_node]) - end - - ### below constraints depending on the layer (every constraint up to the previous layer) - for k_in in 1:k - for node_in in 1:node_count[k_in] - if k_in >= 2 - temp_sum = sum(W[k_in-1][node_in, j] * x[k_in-1-1, j] for j in 1:node_count[k_in-1]) - @constraint(model, x[k_in-1, node_in] <= U[k_in-1, node_in] * z[k_in-1, node_in]) - @constraint(model, s[k_in-1, node_in] <= -L[k_in-1, node_in] * (1 - z[k_in-1, node_in])) - if k_in <= K - 1 - @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in] - s[k_in-1, node_in]) - else # k_in == K - @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in]) - end - end - end - end - - ### below constraints depending on the node - temp_sum = sum(W[k][curr_node, j] * x[k-1, j] for j in 1:node_count[k]) # NOTE! prev layer [k] - if k <= K - 1 - @constraint(model, node_con, temp_sum + b[k][curr_node] == x[k, curr_node] - s[k, curr_node]) - @constraint(model, node_U, x[k, curr_node] <= U[k, curr_node] * z[k, curr_node]) - @constraint(model, node_L, s[k, curr_node] <= -L[k, curr_node] * (1 - z[k, curr_node])) - elseif k == K # == last value of k - @constraint(model, node_con, temp_sum + b[k][curr_node] == x[k, curr_node]) - @constraint(model, node_L, L[k, curr_node] <= x[k, curr_node]) - @constraint(model, node_U, x[k, curr_node] <= U[k, curr_node]) - end - - if obj_function == 1 && k <= K - 1 # Min, hidden layer - @objective(model, Min, x[k, curr_node] - s[k, curr_node]) - elseif obj_function == 2 && k <= K - 1 # Max, hidden layer - @objective(model, Max, x[k, curr_node] - s[k, curr_node]) - elseif obj_function == 1 && k == K # Min, last layer - @objective(model, Min, x[k, curr_node]) - elseif obj_function == 2 && k == K # Max, last layer - @objective(model, Max, x[k, curr_node]) - end - - solve_time = @elapsed optimize!(model) - solve_time = round(solve_time; sigdigits = 3) - # @assert termination_status(model) == OPTIMAL || termination_status(model) == TIME_LIMIT - # "Problem (layer $k (from 1:$K), node $curr_node, $(obj_function == 1 ? "L" : "U")-bound) is infeasible." - optimal = objective_value(model) - - # @show termination_status(model) - # if termination_status(model) == OPTIMAL - # optimal = objective_value(model) - # else - # optimal = Inf - # end - - - println("Thread: $(Threads.threadid()), layer $k, node $curr_node, $(obj_function == 1 ? "L" : "U")-bound: solve time $(solve_time)s, optimal value $(optimal)") - - # fix the model variable L or U corresponding to the current node to be the optimal value - Threads.lock(lock) do - if obj_function == 1 && optimal != Inf # Min and we recieved a new bound - - curr_L_bounds[curr_node_index] = optimal - fix(L[k, curr_node], optimal) - - elseif obj_function == 2 && optimal != Inf # Max and we recieved a new bound - - curr_U_bounds[curr_node_index] = optimal - fix(U[k, curr_node], optimal) - - end - end - - end - - end - - println("Solving optimal constraint bounds using threads complete") - - return curr_U_bounds, curr_L_bounds -end - -""" -bound_tightening_workers(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false, tl::Float64=1.0) - -A multi-threaded (using workers) implementation of optimal tightened constraint bounds L and U for for a trained DNN. -Using these bounds with the create_JuMP_model function reduces solution time for optimization problems. - -# Arguments -- `DNN::Chain`: A trained ReLU DNN. -- `init_U_bounds::Vector{Float32}`: Initial upper bounds on the node values of the DNN. -- `init_L_bounds::Vector{Float32}`: Initial lower bounds on the node values of the DNN. -- `verbose::Bool=false`: Controls Gurobi logs. -- `tl::Float64=1.0`: Controls the time limit for solvign the subproblems - -# Examples -```julia -L_bounds_workers, U_bounds_workers = bound_tightening_workers(DNN, init_U_bounds, init_L_bounds, false, 1.0) -``` -""" -function bound_tightening_workers(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false, tl::Float64=1.0) - - K = length(DNN) # NOTE! there are K+1 layers in the nn - - # store the DNN weights and biases - DNN_params = params(DNN) - W = [DNN_params[2*i-1] for i in 1:K] - b = [DNN_params[2*i] for i in 1:K] - - # stores the node count of layer k (starting at layer k=0) at index k+1 - input_node_count = length(DNN_params[1][1, :]) - node_count = [if k == 1 input_node_count else length(DNN_params[2*(k-1)]) end for k in 1:K+1] - - # store the current optimal bounds in the algorithm - curr_U_bounds = copy(init_U_bounds) - curr_L_bounds = copy(init_L_bounds) - - for k in 1:K - - # Distributed.pmap returns the bounds in order - L_U_bounds = Distributed.pmap(node -> bt_workers_inner(K, k, node, W, b, node_count, curr_U_bounds, curr_L_bounds, verbose, tl), 1:(2*node_count[k+1])) - - for node in 1:node_count[k+1] - prev_layers_node_sum = 0 - for prev_layer in 0:k-1 - prev_layers_node_sum += node_count[prev_layer+1] - end - - # loops nodes twice: 1st time with obj function Min, 2nd time with Max - curr_node = node - obj_function = 1 - if node > node_count[k+1] - curr_node = node - node_count[k+1] - obj_function = 2 - end - curr_node_index = prev_layers_node_sum + curr_node - - # L-bounds in 1:node_count[k+1], U-bounds in 1:(node + node_count[k+1]) - curr_L_bounds[curr_node_index] = L_U_bounds[node] - curr_U_bounds[curr_node_index] = L_U_bounds[node + node_count[k+1]] - end - - end - - println("Solving optimal constraint bounds using workers complete") - - return curr_U_bounds, curr_L_bounds -end - -# Inner function to bound_tightening_workers: assigns a JuMP model to the current worker - -function bt_workers_inner( - K::Int64, - k::Int64, - node::Int64, - W::Vector{Matrix{Float32}}, - b::Vector{Vector{Float32}}, - node_count::Vector{Int64}, - curr_U_bounds::Vector{Float32}, - curr_L_bounds::Vector{Float32}, - verbose::Bool, - tl::Float64 - ) - - model = Model(optimizer_with_attributes(Gurobi.Optimizer, "OutputFlag" => (verbose ? 1 : 0), "Threads" => 1, "TimeLimit" => tl)) - - # keeps track of the current node index starting from layer 1 (out of 0:K) - prev_layers_node_sum = 0 - for prev_layer in 0:k-1 - prev_layers_node_sum += node_count[prev_layer+1] - end - - # loops nodes twice: 1st time with obj function Min, 2nd time with Max - curr_node = node - obj_function = 1 - if node > node_count[k+1] - curr_node = node - node_count[k+1] - obj_function = 2 - end - - # NOTE! below variables and constraints for all opt problems - @variable(model, x[k in 0:K, j in 1:node_count[k+1]] >= 0) - @variable(model, s[k in 1:K-1, j in 1:node_count[k+1]] >= 0) - @variable(model, z[k in 1:K-1, j in 1:node_count[k+1]], Bin) - @variable(model, U[k in 0:K, j in 1:node_count[k+1]]) - @variable(model, L[k in 0:K, j in 1:node_count[k+1]]) - - # fix values to all U[k,j] and L[k,j] from U_bounds and L_bounds - index = 1 - for k in 0:K - for j in 1:node_count[k+1] - fix(U[k, j], curr_U_bounds[index], force=true) - fix(L[k, j], curr_L_bounds[index], force=true) - index += 1 - end - end - - # input layer (layer 0) node bounds are given beforehand - for input_node in 1:node_count[1] - delete_lower_bound(x[0, input_node]) - @constraint(model, L[0, input_node] <= x[0, input_node]) - @constraint(model, x[0, input_node] <= U[0, input_node]) - end - - # deleting lower bound for output nodes - for output_node in 1:node_count[K+1] - delete_lower_bound(x[K, output_node]) - end - - ### below constraints depending on the layer (every constraint up to the previous layer) - for k_in in 1:k - for node_in in 1:node_count[k_in] - if k_in >= 2 - temp_sum = sum(W[k_in-1][node_in, j] * x[k_in-1-1, j] for j in 1:node_count[k_in-1]) - @constraint(model, x[k_in-1, node_in] <= U[k_in-1, node_in] * z[k_in-1, node_in]) - @constraint(model, s[k_in-1, node_in] <= -L[k_in-1, node_in] * (1 - z[k_in-1, node_in])) - if k_in <= K - 1 - @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in] - s[k_in-1, node_in]) - else # k_in == K - @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in]) - end - end - end - end - - ### below constraints depending on the node - temp_sum = sum(W[k][curr_node, j] * x[k-1, j] for j in 1:node_count[k]) # NOTE! prev layer [k] - if k <= K - 1 - @constraint(model, node_con, temp_sum + b[k][curr_node] == x[k, curr_node] - s[k, curr_node]) - @constraint(model, node_U, x[k, curr_node] <= U[k, curr_node] * z[k, curr_node]) - @constraint(model, node_L, s[k, curr_node] <= -L[k, curr_node] * (1 - z[k, curr_node])) - elseif k == K # == last value of k - @constraint(model, node_con, temp_sum + b[k][curr_node] == x[k, curr_node]) - @constraint(model, node_L, L[k, curr_node] <= x[k, curr_node]) - @constraint(model, node_U, x[k, curr_node] <= U[k, curr_node]) - end - - if obj_function == 1 && k <= K - 1 # Min, hidden layer - @objective(model, Min, x[k, curr_node] - s[k, curr_node]) - elseif obj_function == 2 && k <= K - 1 # Max, hidden layer - @objective(model, Max, x[k, curr_node] - s[k, curr_node]) - elseif obj_function == 1 && k == K # Min, last layer - @objective(model, Min, x[k, curr_node]) - elseif obj_function == 2 && k == K # Max, last layer - @objective(model, Max, x[k, curr_node]) - end - - solve_time = @elapsed optimize!(model) - solve_time = round(solve_time; sigdigits = 3) - # @assert termination_status(model) == OPTIMAL || termination_status(model) == TIME_LIMIT - # "Problem (layer $k (from 1:$K), node $curr_node, $(obj_function == 1 ? "L" : "U")-bound) is infeasible." - optimal = objective_value(model) - println("Worker: $(myid()), layer $k, node $curr_node, $(obj_function == 1 ? "L" : "U")-bound: solve time $(solve_time)s, optimal value $(optimal)") - - return optimal -end - -""" -bound_tightening_2workers(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false) - -A multi-threaded (using workers) implementation of optimal tightened constraint bounds L and U for for a trained DNN. -This function uses two in-place models at each layer to reduce memory usage. A max of 2 workers in use simultaneously. -Using these bounds with the create_JuMP_model function reduces solution time for optimization problems. - -# Arguments -- `DNN::Chain`: A trained ReLU DNN. -- `init_U_bounds::Vector{Float32}`: Initial upper bounds on the node values of the DNN. -- `init_L_bounds::Vector{Float32}`: Initial lower bounds on the node values of the DNN. -- `verbose::Bool=false`: Controls Gurobi logs. -- `tl::Float64=1.0`: Controls the time limit for solvign the subproblems. - -# Examples -```julia -L_bounds_workers, U_bounds_workers = bound_tightening_2workers(DNN, init_U_bounds, init_L_bounds, false, 1.0) -``` -""" -function bound_tightening_2workers(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false, tl::Float64=1.0) - - K = length(DNN) # NOTE! there are K+1 layers in the nn - - # store the DNN weights and biases - DNN_params = params(DNN) - W = [DNN_params[2*i-1] for i in 1:K] - b = [DNN_params[2*i] for i in 1:K] - - # stores the node count of layer k (starting at layer k=0) at index k+1 - input_node_count = length(DNN_params[1][1, :]) - node_count = [if k == 1 input_node_count else length(DNN_params[2*(k-1)]) end for k in 1:K+1] - - # store the current optimal bounds in the algorithm - curr_U_bounds = copy(init_U_bounds) - curr_L_bounds = copy(init_L_bounds) - - # split the available threads into 2 to be assigned to each worker (integer division) - n = Threads.nthreads() - threads_split = [n÷2, n-(n÷2)] - - for k in 1:K - - L_U_bounds = Distributed.pmap(obj_function -> - bt_2workers_inner(K, k, obj_function, W, b, node_count, curr_U_bounds, curr_L_bounds, threads_split[obj_function], verbose, tl), 1:2) - - curr_L_bounds = L_U_bounds[1] - curr_U_bounds = L_U_bounds[2] - - end - - println("Solving optimal constraint bounds complete") - - return curr_U_bounds, curr_L_bounds -end - - -# Inner function to solve_optimal_bounds_2workers: solves L or U bounds for all nodes in a layer using the same JuMP model - -function bt_2workers_inner( - K::Int64, - k::Int64, - obj_function::Int64, - W::Vector{Matrix{Float32}}, - b::Vector{Vector{Float32}}, - node_count::Vector{Int64}, - curr_U_bounds::Vector{Float32}, - curr_L_bounds::Vector{Float32}, - n_threads::Int64, - verbose::Bool, - tl::Float64 - ) - - curr_U_bounds_copy = copy(curr_U_bounds) - curr_L_bounds_copy = copy(curr_L_bounds) - - model = Model(optimizer_with_attributes(Gurobi.Optimizer, "OutputFlag" => (verbose ? 1 : 0), "Threads" => n_threads, "TimeLimit" => tl)) - - # NOTE! below variables and constraints for all opt problems - @variable(model, x[k in 0:K, j in 1:node_count[k+1]] >= 0) - @variable(model, s[k in 1:K-1, j in 1:node_count[k+1]] >= 0) - @variable(model, z[k in 1:K-1, j in 1:node_count[k+1]], Bin) - @variable(model, U[k in 0:K, j in 1:node_count[k+1]]) - @variable(model, L[k in 0:K, j in 1:node_count[k+1]]) - - # fix values to all U[k,j] and L[k,j] from U_bounds and L_bounds - index = 1 - for k in 0:K - for j in 1:node_count[k+1] - fix(U[k, j], curr_U_bounds[index], force=true) - fix(L[k, j], curr_L_bounds[index], force=true) - index += 1 - end - end - - # input layer (layer 0) node bounds are given beforehand - for input_node in 1:node_count[1] - delete_lower_bound(x[0, input_node]) - @constraint(model, L[0, input_node] <= x[0, input_node]) - @constraint(model, x[0, input_node] <= U[0, input_node]) - end - - # deleting lower bound for output nodes - for output_node in 1:node_count[K+1] - delete_lower_bound(x[K, output_node]) - end - - ### below constraints depending on the layer (every constraint up to the previous layer) - for k_in in 1:k - for node_in in 1:node_count[k_in] - if k_in >= 2 - temp_sum = sum(W[k_in-1][node_in, j] * x[k_in-1-1, j] for j in 1:node_count[k_in-1]) - @constraint(model, x[k_in-1, node_in] <= U[k_in-1, node_in] * z[k_in-1, node_in]) - @constraint(model, s[k_in-1, node_in] <= -L[k_in-1, node_in] * (1 - z[k_in-1, node_in])) - if k_in <= K - 1 - @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in] - s[k_in-1, node_in]) - else # k_in == K - @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in]) - end - end - end - end - - for node in 1:node_count[k+1] - - prev_layers_node_sum = 0 - for prev_layer in 0:k-1 - prev_layers_node_sum += node_count[prev_layer+1] - end - curr_node_index = prev_layers_node_sum + node - - ### below constraints depending on the node - temp_sum = sum(W[k][node, j] * x[k-1, j] for j in 1:node_count[k]) # NOTE! prev layer [k] - if k <= K - 1 - @constraint(model, node_con, temp_sum + b[k][node] == x[k, node] - s[k, node]) - @constraint(model, node_U, x[k, node] <= U[k, node] * z[k, node]) - @constraint(model, node_L, s[k, node] <= -L[k, node] * (1 - z[k, node])) - elseif k == K # == last value of k - @constraint(model, node_con, temp_sum + b[k][node] == x[k, node]) - @constraint(model, node_L, L[k, node] <= x[k, node]) - @constraint(model, node_U, x[k, node] <= U[k, node]) - end - - if obj_function == 1 && k <= K - 1 # Min, hidden layer - @objective(model, Min, x[k, node] - s[k, node]) - elseif obj_function == 2 && k <= K - 1 # Max, hidden layer - @objective(model, Max, x[k, node] - s[k, node]) - elseif obj_function == 1 && k == K # Min, last layer - @objective(model, Min, x[k, node]) - elseif obj_function == 2 && k == K # Max, last layer - @objective(model, Max, x[k, node]) - end - - solve_time = @elapsed optimize!(model) - solve_time = round(solve_time; sigdigits = 3) - @assert termination_status(model) == OPTIMAL || termination_status(model) == TIME_LIMIT - "Problem (layer $k (from 1:$K), node $node, $(obj_function == 1 ? "L" : "U")-bound) is infeasible." - optimal = objective_value(model) - println("Worker: $(myid()), layer $k, node $node, $(obj_function == 1 ? "L" : "U")-bound: solve time $(solve_time)s, optimal value $(optimal)") - - # fix the model variable L or U corresponding to the current node to be the optimal value - if obj_function == 1 # Min - curr_L_bounds_copy[curr_node_index] = optimal - elseif obj_function == 2 # Max - curr_U_bounds_copy[curr_node_index] = optimal - end - - # deleting and unregistering the constraints assigned to the current node - delete(model, node_con) - delete(model, node_L) - delete(model, node_U) - unregister(model, :node_con) - unregister(model, :node_L) - unregister(model, :node_U) - end - - if obj_function == 1 # Min - return curr_L_bounds_copy - elseif obj_function == 2 # Max - return curr_U_bounds_copy - end - -end \ No newline at end of file diff --git a/src/neural_networks/bound_tightening_new.jl b/src/neural_networks/bound_tightening_new.jl deleted file mode 100644 index 569bb37..0000000 --- a/src/neural_networks/bound_tightening_new.jl +++ /dev/null @@ -1,162 +0,0 @@ -using BSON -using Flux -using JuMP -using Gurobi -using SharedArrays -using Distributed - -function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}; tighten_bounds=true, distributed=false) - - K = length(NN_model) # number of layers (input layer not included) - W = [Flux.params(NN_model)[2*k-1] for k in 1:K] - b = [Flux.params(NN_model)[2*k] for k in 1:K] - - input_length = Int((length(W[1]) / length(b[1]))) - neuron_count = [length(b[k]) for k in eachindex(b)] - neurons(layer) = layer == 0 ? [i for i in 1:input_length] : [i for i in 1:neuron_count[layer]] - - @assert input_length == length(init_ub) == length(init_lb) "Initial bounds arrays must be the same length as the input layer" - - # build model up to second layer - jump_model = Model() - set_optimizer(jump_model, () -> Gurobi.Optimizer(ENV[myid()])) - set_silent(jump_model) - set_attribute(jump_model, "TimeLimit", 0.1) - - @variable(jump_model, x[layer = 0:K, neurons(layer)]) - @variable(jump_model, s[layer = 0:K, neurons(layer)]) - @variable(jump_model, z[layer = 0:K, neurons(layer)]) - - @constraint(jump_model, [j = 1:input_length], x[0, j] <= init_ub[j]) - @constraint(jump_model, [j = 1:input_length], x[0, j] >= init_lb[j]) - - bounds_x = Vector{Vector}(undef, K) - bounds_s = Vector{Vector}(undef, K) - - for layer in 1:K # hidden layers to output layer - second layer and up - - ub_x = fill(1000.0, length(neurons(layer))) |> SharedArray - ub_s = fill(1000.0, length(neurons(layer))) |> SharedArray - - # TODO: For parallelization the model must be copied for each neuron in a new layer to prevent data races - - println("\nLAYER $layer") - - if tighten_bounds - if distributed - @sync @distributed for neuron in 1:neuron_count[layer] - ub_x[neuron], ub_s[neuron] = calculate_bounds_copy(jump_model, layer, neuron, W, b, neurons) - end - else - for neuron in 1:neuron_count[layer] - print("$neuron ") - ub_x[neuron], ub_s[neuron] = calculate_bounds(jump_model, layer, neuron, W, b, neurons) - end - end - end - - for neuron in 1:neuron_count[layer] - - @constraint(jump_model, x[layer, neuron] >= 0) - @constraint(jump_model, s[layer, neuron] >= 0) - set_binary(z[layer, neuron]) - - @constraint(jump_model, z[layer, neuron] --> {x[layer, neuron] <= 0}) - @constraint(jump_model, !z[layer, neuron] --> {s[layer, neuron] <= 0}) - - @constraint(jump_model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1, i] for i in neurons(layer-1))) - - end - - bounds_x[layer] = ub_x - bounds_s[layer] = ub_s - - @constraint(jump_model, [neuron = neurons(layer)], x[layer, neuron] <= ub_x[neuron]) - @constraint(jump_model, [neuron = neurons(layer)], s[layer, neuron] <= ub_s[neuron]) - end - - return jump_model, bounds_x, bounds_s -end - -function calculate_bounds(model::JuMP.Model, layer, neuron, W, b, neurons) - - x = model[:x] - s = model[:s] - z = model[:z] - - @constraint(model, x_con, x[layer, neuron] >= 0) - @constraint(model, s_con, s[layer, neuron] >= 0) - set_binary(z[layer, neuron]) - - @constraint(model, zx_con, z[layer, neuron] --> {x[layer, neuron] <= 0}) - @constraint(model, zs_con, !z[layer, neuron] --> {s[layer, neuron] <= 0}) - - @constraint(model, w_con, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1, i] for i in neurons(layer-1))) - - @objective(model, Max, x[layer, neuron]) - optimize!(model) - @assert primal_status(model) == MOI.FEASIBLE_POINT "No solution found in time limit." - ub_x = objective_bound(model) - - @objective(model, Max, s[layer, neuron]) - optimize!(model) - @assert primal_status(model) == MOI.FEASIBLE_POINT "No solution found in time limit." - ub_s = objective_bound(model) - - delete(model, x_con) - delete(model, s_con) - delete(model, zx_con) - delete(model, zs_con) - delete(model, w_con) - unregister(model, :x_con) - unregister(model, :s_con) - unregister(model, :zx_con) - unregister(model, :zs_con) - unregister(model, :w_con) - unset_binary(z[layer, neuron]) - - return ub_x, ub_s -end - -function calculate_bounds_copy(input_model::JuMP.Model, layer, neuron, W, b, neurons) - - model = copy(input_model) - set_optimizer(model, () -> Gurobi.Optimizer(ENV[myid()])) - set_silent(model) - - x = model[:x] - s = model[:s] - z = model[:z] - - @constraint(model, x[layer, neuron] >= 0) - @constraint(model, s[layer, neuron] >= 0) - set_binary(z[layer, neuron]) - - @constraint(model, z[layer, neuron] --> {x[layer, neuron] <= 0}) - @constraint(model, !z[layer, neuron] --> {s[layer, neuron] <= 0}) - - @constraint(model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1, i] for i in neurons(layer-1))) - - @objective(model, Max, x[layer, neuron]) - optimize!(model) - @assert primal_status(model) == MOI.FEASIBLE_POINT "No solution found in time limit." - ub_x = objective_bound(model) - - @objective(model, Max, s[layer, neuron]) - optimize!(model) - @assert primal_status(model) == MOI.FEASIBLE_POINT "No solution found in time limit." - ub_s = objective_bound(model) - - return ub_x, ub_s -end - -function forward_pass!(jump_model::JuMP.Model, input::Vector{Float32}) - @assert length(input) == length(jump_model[:x][0, :]) "Incorrect input length." - - [fix(jump_model[:x][0, i], input[i], force=true) for i in eachindex(input)] - optimize!(jump_model) - - (last_layer, outputs) = maximum(keys(jump_model[:x].data)) - result = value.(jump_model[:x][last_layer, :]) - return [result[i] for i in 1:outputs] -end \ No newline at end of file diff --git a/src/neural_networks/pretrained_model_pruner.jl b/src/neural_networks/pretrained_model_pruner.jl index c263b1b..8d50d03 100644 --- a/src/neural_networks/pretrained_model_pruner.jl +++ b/src/neural_networks/pretrained_model_pruner.jl @@ -114,7 +114,12 @@ function find_lp_bounds(n_neurons, parent_dir) U_bounds = Float64[-0.5, 0.5, [1000000 for _ in 3:n_neurons_initial]...] L_bounds = Float64[-1.5, -0.5, [-1000000 for _ in 3:n_neurons_initial]...] - best_LP_U_bounds, best_LP_L_bounds = bound_tightening(model, U_bounds, L_bounds, false, true, gurobi_env) + #best_LP_U_bounds, best_LP_L_bounds = bound_tightening(model, U_bounds, L_bounds, false, true, gurobi_env) + _, upper_bounds, lower_bounds = NN_to_MIP(model, [-0.5, 0.5], [-1.0, -1.0]; tighten_bounds=true); + + # TODO add initial bounds to the beginning and the last layer bounds + best_LP_U_bounds = collect(Iterators.flatten(upper_bounds)) + best_LP_L_bounds = collect(Iterators.flatten(lower_bounds)) npzwrite(joinpath(subdir_path, "upper_lp.npy"), best_LP_U_bounds) npzwrite(joinpath(subdir_path, "lower_lp.npy"), best_LP_L_bounds) From 878792ce7259d4396575083605a39d25f04d05a7 Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Tue, 30 Jan 2024 16:55:50 +0200 Subject: [PATCH 13/32] icorporated new bound tightening into package --- src/Gogeta.jl | 15 ++---- src/neural_networks/NN_test_serra.jl | 52 +++++++++++-------- src/neural_networks/bound_tightening_serra.jl | 15 ++++-- .../bound_tightening_serra_parallel.jl | 22 +++++--- 4 files changed, 59 insertions(+), 45 deletions(-) diff --git a/src/Gogeta.jl b/src/Gogeta.jl index 7df49c1..4576f10 100644 --- a/src/Gogeta.jl +++ b/src/Gogeta.jl @@ -1,16 +1,9 @@ module Gogeta -include("neural_networks/JuMP_model.jl") -export create_JuMP_model, evaluate! +include("neural_networks/bound_tightening_serra.jl") +export NN_to_MIP, forward_pass!, SolverParams -include("neural_networks/CNN_JuMP_model.jl") -export create_CNN_JuMP_model, evaluate_CNN! - -include("neural_networks/bound_tightening.jl") -export bound_tightening, - bound_tightening_threads, - bound_tightening_workers, - bound_tightening_2workers +include("neural_networks/bound_tightening_serra_parallel.jl") include("tree_ensembles/types.jl") export TEModel, extract_evotrees_info @@ -21,4 +14,4 @@ export get_solution include("tree_ensembles/TE_to_MIP.jl") export TE_to_MIP, optimize_with_initial_constraints!, optimize_with_lazy_constraints! -end +end \ No newline at end of file diff --git a/src/neural_networks/NN_test_serra.jl b/src/neural_networks/NN_test_serra.jl index 5a11f45..373ccd3 100644 --- a/src/neural_networks/NN_test_serra.jl +++ b/src/neural_networks/NN_test_serra.jl @@ -3,38 +3,46 @@ x_train = data[:, 1:750]; y_train = [sum(x_train[:, col].^2) for col in 1:750]; using Distributed - -include("bound_tightening_serra.jl") +using Flux +using Random addprocs(7) -@everywhere include("bound_tightening_serra_parallel.jl") -@everywhere GUROBI_ENV = [Gurobi.Env() for i in 1:nprocs()]; -@everywhere SILENT = true; -@everywhere LIMIT = 0; -@everywhere RELAX = true; -@everywhere THREADS = 0; +@everywhere using Gogeta -using Random -Random.seed!(1234); - -model = Chain( - Dense(2 => 10, relu), - Dense(10 => 30, relu), - Dense(30 => 30, relu), - Dense(30 => 20, relu), - Dense(20 => 5, relu), - Dense(5 => 1) -); - -@time jump_model, upper_bounds, lower_bounds = NN_to_MIP(model, [1.0, 1.0], [-1.0, -1.0]; tighten_bounds=true); +begin + Random.seed!(1234); + + model = Chain( + Dense(2 => 10, relu), + Dense(10 => 50, relu), + Dense(50 => 20, relu), + Dense(20 => 5, relu), + Dense(5 => 1) + ) +end + +solver_params = SolverParams(true, 0, true, 0) + +@time jump_model, upper_bounds, lower_bounds = NN_to_MIP(model, [1.0, 1.0], [-1.0, -1.0], solver_params; tighten_bounds=true); +@time jump_model_relax, upper_bounds_relax, lower_bounds_relax = NN_to_MIP(model, [1.0, 1.0], [-1.0, -1.0], solver_params; tighten_bounds=true); vec(model(x_train)) ≈ [forward_pass!(jump_model, x_train[:, i])[1] for i in 1:750] +vec(model(x_train)) ≈ [forward_pass!(jump_model_relax, x_train[:, i])[1] for i in 1:750] + +rmprocs(workers()) include("bound_tightening.jl") n_neurons = 2 + sum(map(x -> length(x), [Flux.params(model)[2*k] for k in 1:length(model)])); @time U, L = bound_tightening(model, [i<=2 ? 1.0 : 1000.0 for i in 1:n_neurons], [i<=2 ? -1.0 : -1000.0 for i in 1:n_neurons]) +@time U_rel, L_rel = bound_tightening(model, [i<=2 ? 1.0 : 1000.0 for i in 1:n_neurons], [i<=2 ? -1.0 : -1000.0 for i in 1:n_neurons], false, true) using Plots plot(collect(Iterators.flatten(upper_bounds)) .- U[3:end-1]) -plot!(collect(Iterators.flatten(lower_bounds)) .+ L[3:end-1]) \ No newline at end of file +plot!(collect(Iterators.flatten(lower_bounds)) .+ L[3:end-1]) + +plot(collect(Iterators.flatten(upper_bounds_relax)) .- U_rel[3:end-1]) +plot!(collect(Iterators.flatten(lower_bounds_relax)) .+ L_rel[3:end-1]) + +plot(collect(Iterators.flatten(upper_bounds_relax)) - collect(Iterators.flatten(upper_bounds))) +plot!(collect(Iterators.flatten(lower_bounds_relax)) - collect(Iterators.flatten(lower_bounds))) \ No newline at end of file diff --git a/src/neural_networks/bound_tightening_serra.jl b/src/neural_networks/bound_tightening_serra.jl index 281d0d1..60f2452 100644 --- a/src/neural_networks/bound_tightening_serra.jl +++ b/src/neural_networks/bound_tightening_serra.jl @@ -2,7 +2,14 @@ using Flux using JuMP using Distributed -function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}; tighten_bounds=false, big_M=1000.0) +struct SolverParams + silent::Bool + threads::Int + relax::Bool + time_limit::Float64 +end + +function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}, solver_params::SolverParams; tighten_bounds::Bool=false, big_M::Float64=1000.0) K = length(NN_model) # number of layers (input layer not included) @assert reduce(&, [NN_model[i].σ == relu for i in 1:K-1]) "Neural network must use the relu activation function." @@ -16,10 +23,10 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect neurons(layer) = layer == 0 ? [i for i in 1:input_length] : [i for i in 1:neuron_count[layer]] @assert input_length == length(init_ub) == length(init_lb) "Initial bounds arrays must be the same length as the input layer" - + # build model up to second layer jump_model = Model() - set_solver_params!(jump_model) + set_solver_params!(jump_model, solver_params) @variable(jump_model, x[layer = 0:K, neurons(layer)]) @variable(jump_model, s[layer = 0:K-1, neurons(layer)]) @@ -39,7 +46,7 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect println("\nLAYER $layer") if tighten_bounds - bounds = pmap(neuron -> calculate_bounds(copy_model(jump_model), layer, neuron, W, b, neurons), neurons(layer)) + bounds = pmap(neuron -> calculate_bounds(copy_model(jump_model, solver_params), layer, neuron, W, b, neurons), neurons(layer)) bounds_U[layer], bounds_L[layer] = [bound[1] for bound in bounds], [bound[2] for bound in bounds] end diff --git a/src/neural_networks/bound_tightening_serra_parallel.jl b/src/neural_networks/bound_tightening_serra_parallel.jl index 9d59946..cde084e 100644 --- a/src/neural_networks/bound_tightening_serra_parallel.jl +++ b/src/neural_networks/bound_tightening_serra_parallel.jl @@ -1,18 +1,24 @@ using JuMP using Gurobi -function copy_model(input_model) +const GUROBI_ENV = Ref{Gurobi.Env}() + +function __init__() + const GUROBI_ENV[] = Gurobi.Env() +end + +function copy_model(input_model, solver_params) model = copy(input_model) - set_solver_params!(model) + set_solver_params!(model, solver_params) return model end -function set_solver_params!(model) - set_optimizer(model, () -> Gurobi.Optimizer(GUROBI_ENV[myid()])) - SILENT && set_silent(model) - THREADS != 0 && set_attribute(model, "Threads", THREADS) - RELAX && relax_integrality(model) - LIMIT != 0 && set_attribute(model, "TimeLimit", LIMIT) +function set_solver_params!(model, params) + set_optimizer(model, () -> Gurobi.Optimizer(GUROBI_ENV[])) + params.silent && set_silent(model) + params.threads != 0 && set_attribute(model, "Threads", params.threads) + params.relax && relax_integrality(model) + params.time_limit != 0 && set_attribute(model, "TimeLimit", params.time_limit) end function calculate_bounds(model::JuMP.Model, layer, neuron, W, b, neurons) From 3b8b3264f094a0b262d78cdb54932c49d12312a2 Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Wed, 31 Jan 2024 14:08:03 +0200 Subject: [PATCH 14/32] minor improvements to bound tightening --- Project.toml | 1 + src/neural_networks/NN_test_serra.jl | 15 ++++++++++++++- src/neural_networks/bound_tightening_serra.jl | 8 ++++++-- .../bound_tightening_serra_parallel.jl | 18 ++++++++++++++---- src/neural_networks/pretrained_model_pruner.jl | 2 +- 5 files changed, 36 insertions(+), 8 deletions(-) diff --git a/Project.toml b/Project.toml index c6165bb..7a6008b 100644 --- a/Project.toml +++ b/Project.toml @@ -18,6 +18,7 @@ NPZ = "15e1cf62-19b3-5cfa-8e77-841668bca605" PProf = "e4faabce-9ead-11e9-39d9-4379958e3056" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Revise = "295af30f-e4ad-537b-8983-00126c2a3abe" SCIP = "82193955-e24f-5292-bf16-6f2c5261a85f" SharedArrays = "1a1011a3-84de-559e-8e89-a11a2f7dc383" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" diff --git a/src/neural_networks/NN_test_serra.jl b/src/neural_networks/NN_test_serra.jl index 373ccd3..795755c 100644 --- a/src/neural_networks/NN_test_serra.jl +++ b/src/neural_networks/NN_test_serra.jl @@ -5,6 +5,8 @@ y_train = [sum(x_train[:, col].^2) for col in 1:750]; using Distributed using Flux using Random +using Revise +using Gogeta addprocs(7) @@ -22,7 +24,18 @@ begin ) end -solver_params = SolverParams(true, 0, true, 0) +begin + Random.seed!(1234); + + model = Chain( + Dense(2 => 10, relu), + Dense(10 => 20, relu), + Dense(20 => 5, relu), + Dense(5 => 1) + ) +end + +solver_params = SolverParams(true, 0, false, 0) @time jump_model, upper_bounds, lower_bounds = NN_to_MIP(model, [1.0, 1.0], [-1.0, -1.0], solver_params; tighten_bounds=true); @time jump_model_relax, upper_bounds_relax, lower_bounds_relax = NN_to_MIP(model, [1.0, 1.0], [-1.0, -1.0], solver_params; tighten_bounds=true); diff --git a/src/neural_networks/bound_tightening_serra.jl b/src/neural_networks/bound_tightening_serra.jl index 60f2452..e15f336 100644 --- a/src/neural_networks/bound_tightening_serra.jl +++ b/src/neural_networks/bound_tightening_serra.jl @@ -46,8 +46,12 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect println("\nLAYER $layer") if tighten_bounds - bounds = pmap(neuron -> calculate_bounds(copy_model(jump_model, solver_params), layer, neuron, W, b, neurons), neurons(layer)) - bounds_U[layer], bounds_L[layer] = [bound[1] for bound in bounds], [bound[2] for bound in bounds] + bounds = if nprocs() > 1 + pmap(neuron -> calculate_bounds(copy_model(jump_model, solver_params), layer, neuron, W, b, neurons), neurons(layer)) + else + map(neuron -> calculate_bounds(jump_model, layer, neuron, W, b, neurons), neurons(layer)) + end + bounds_U[layer], bounds_L[layer] = [bound[1] > big_M ? big_M : bound[1] for bound in bounds], [bound[2] > big_M ? big_M : bound[2] for bound in bounds] end for neuron in 1:neuron_count[layer] diff --git a/src/neural_networks/bound_tightening_serra_parallel.jl b/src/neural_networks/bound_tightening_serra_parallel.jl index cde084e..64690aa 100644 --- a/src/neural_networks/bound_tightening_serra_parallel.jl +++ b/src/neural_networks/bound_tightening_serra_parallel.jl @@ -1,5 +1,6 @@ using JuMP using Gurobi +using GLPK const GUROBI_ENV = Ref{Gurobi.Env}() @@ -15,24 +16,33 @@ end function set_solver_params!(model, params) set_optimizer(model, () -> Gurobi.Optimizer(GUROBI_ENV[])) + # set_optimizer(model, () -> GLPK.Optimizer()) params.silent && set_silent(model) params.threads != 0 && set_attribute(model, "Threads", params.threads) params.relax && relax_integrality(model) params.time_limit != 0 && set_attribute(model, "TimeLimit", params.time_limit) + # params.time_limit != 0 && set_attribute(model, "tm_lim", params.time_limit) end function calculate_bounds(model::JuMP.Model, layer, neuron, W, b, neurons) @objective(model, Max, b[layer][neuron] + sum(W[layer][neuron, i] * model[:x][layer-1, i] for i in neurons(layer-1))) optimize!(model) - @assert primal_status(model) == MOI.FEASIBLE_POINT "No solution found in time limit." - upper_bound = max(objective_bound(model), 0.0) + upper_bound = if termination_status(model) == OPTIMAL + max(objective_value(model), 0.0) + else + max(objective_bound(model), 0.0) + end set_objective_sense(model, MIN_SENSE) optimize!(model) - @assert primal_status(model) == MOI.FEASIBLE_POINT "No solution found in time limit." - lower_bound = max(-objective_bound(model), 0.0) + + lower_bound = if termination_status(model) == OPTIMAL + max(-objective_value(model), 0.0) + else + max(-objective_bound(model), 0.0) + end println("Neuron: $neuron") diff --git a/src/neural_networks/pretrained_model_pruner.jl b/src/neural_networks/pretrained_model_pruner.jl index 8d50d03..299ad9d 100644 --- a/src/neural_networks/pretrained_model_pruner.jl +++ b/src/neural_networks/pretrained_model_pruner.jl @@ -115,7 +115,7 @@ function find_lp_bounds(n_neurons, parent_dir) L_bounds = Float64[-1.5, -0.5, [-1000000 for _ in 3:n_neurons_initial]...] #best_LP_U_bounds, best_LP_L_bounds = bound_tightening(model, U_bounds, L_bounds, false, true, gurobi_env) - _, upper_bounds, lower_bounds = NN_to_MIP(model, [-0.5, 0.5], [-1.0, -1.0]; tighten_bounds=true); + _, upper_bounds, lower_bounds = NN_to_MIP(model, [-0.5, 0.5], [-1.5, -0.5], SolverParams(true, 0, true, 0); tighten_bounds=true); # TODO add initial bounds to the beginning and the last layer bounds best_LP_U_bounds = collect(Iterators.flatten(upper_bounds)) From 05c529058ff12607f8eda5adc321399d17783138 Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Thu, 1 Feb 2024 15:12:26 +0200 Subject: [PATCH 15/32] reorganized NN code --- .../NN_test_serra.jl => examples/NN_test.jl | 24 ++++++++++++++----- src/Gogeta.jl | 4 ++-- ...bound_tightening_serra.jl => NN_to_MIP.jl} | 12 ++++++---- ...tightening_serra_parallel.jl => bounds.jl} | 0 src/neural_networks/{ => old}/JuMP_model.jl | 0 .../{ => old}/bound_tightening.jl | 0 6 files changed, 28 insertions(+), 12 deletions(-) rename src/neural_networks/NN_test_serra.jl => examples/NN_test.jl (77%) rename src/neural_networks/{bound_tightening_serra.jl => NN_to_MIP.jl} (92%) rename src/neural_networks/{bound_tightening_serra_parallel.jl => bounds.jl} (100%) rename src/neural_networks/{ => old}/JuMP_model.jl (100%) rename src/neural_networks/{ => old}/bound_tightening.jl (100%) diff --git a/src/neural_networks/NN_test_serra.jl b/examples/NN_test.jl similarity index 77% rename from src/neural_networks/NN_test_serra.jl rename to examples/NN_test.jl index 795755c..6c381ca 100644 --- a/src/neural_networks/NN_test_serra.jl +++ b/examples/NN_test.jl @@ -35,7 +35,16 @@ begin ) end -solver_params = SolverParams(true, 0, false, 0) +begin + Random.seed!(1234); + + model = Chain( + Dense(2 => 3, relu), + Dense(3 => 1) + ) +end + +solver_params = SolverParams(silent=true, threads=0, relax=false, time_limit=0) @time jump_model, upper_bounds, lower_bounds = NN_to_MIP(model, [1.0, 1.0], [-1.0, -1.0], solver_params; tighten_bounds=true); @time jump_model_relax, upper_bounds_relax, lower_bounds_relax = NN_to_MIP(model, [1.0, 1.0], [-1.0, -1.0], solver_params; tighten_bounds=true); @@ -44,18 +53,21 @@ vec(model(x_train)) ≈ [forward_pass!(jump_model_relax, x_train[:, i])[1] for i rmprocs(workers()) -include("bound_tightening.jl") +include("../src/neural_networks/old/bound_tightening.jl") n_neurons = 2 + sum(map(x -> length(x), [Flux.params(model)[2*k] for k in 1:length(model)])); @time U, L = bound_tightening(model, [i<=2 ? 1.0 : 1000.0 for i in 1:n_neurons], [i<=2 ? -1.0 : -1000.0 for i in 1:n_neurons]) @time U_rel, L_rel = bound_tightening(model, [i<=2 ? 1.0 : 1000.0 for i in 1:n_neurons], [i<=2 ? -1.0 : -1000.0 for i in 1:n_neurons], false, true) +all(map(x -> x > last(L)[] && x < last(U)[], model(x_train))) +all(map(x -> x > -last(lower_bounds)[] && x < last(upper_bounds)[], model(x_train))) + using Plots -plot(collect(Iterators.flatten(upper_bounds)) .- U[3:end-1]) -plot!(collect(Iterators.flatten(lower_bounds)) .+ L[3:end-1]) +plot(collect(Iterators.flatten(upper_bounds)) .- U[3:end]) +plot!(collect(Iterators.flatten(lower_bounds)) .+ L[3:end]) -plot(collect(Iterators.flatten(upper_bounds_relax)) .- U_rel[3:end-1]) -plot!(collect(Iterators.flatten(lower_bounds_relax)) .+ L_rel[3:end-1]) +plot(collect(Iterators.flatten(upper_bounds_relax)) .- U_rel[3:end]) +plot!(collect(Iterators.flatten(lower_bounds_relax)) .+ L_rel[3:end]) plot(collect(Iterators.flatten(upper_bounds_relax)) - collect(Iterators.flatten(upper_bounds))) plot!(collect(Iterators.flatten(lower_bounds_relax)) - collect(Iterators.flatten(lower_bounds))) \ No newline at end of file diff --git a/src/Gogeta.jl b/src/Gogeta.jl index 4576f10..2b1daf2 100644 --- a/src/Gogeta.jl +++ b/src/Gogeta.jl @@ -1,9 +1,9 @@ module Gogeta -include("neural_networks/bound_tightening_serra.jl") +include("neural_networks/NN_to_MIP.jl") export NN_to_MIP, forward_pass!, SolverParams -include("neural_networks/bound_tightening_serra_parallel.jl") +include("neural_networks/bounds.jl") include("tree_ensembles/types.jl") export TEModel, extract_evotrees_info diff --git a/src/neural_networks/bound_tightening_serra.jl b/src/neural_networks/NN_to_MIP.jl similarity index 92% rename from src/neural_networks/bound_tightening_serra.jl rename to src/neural_networks/NN_to_MIP.jl index e15f336..a0c098b 100644 --- a/src/neural_networks/bound_tightening_serra.jl +++ b/src/neural_networks/NN_to_MIP.jl @@ -2,7 +2,7 @@ using Flux using JuMP using Distributed -struct SolverParams +@kwdef struct SolverParams silent::Bool threads::Int relax::Bool @@ -35,10 +35,10 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect @constraint(jump_model, [j = 1:input_length], x[0, j] <= init_ub[j]) @constraint(jump_model, [j = 1:input_length], x[0, j] >= init_lb[j]) - bounds_U = Vector{Vector}(undef, K-1) - bounds_L = Vector{Vector}(undef, K-1) + bounds_U = Vector{Vector}(undef, K) + bounds_L = Vector{Vector}(undef, K) - for layer in 1:K-1 # hidden layers + for layer in 1:K # hidden layers and output bounds_U[layer] = fill(big_M, length(neurons(layer))) bounds_L[layer] = fill(big_M, length(neurons(layer))) @@ -54,6 +54,10 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect bounds_U[layer], bounds_L[layer] = [bound[1] > big_M ? big_M : bound[1] for bound in bounds], [bound[2] > big_M ? big_M : bound[2] for bound in bounds] end + if layer == K # output bounds calculated but no unnecessary constraints added + break + end + for neuron in 1:neuron_count[layer] @constraint(jump_model, x[layer, neuron] >= 0) diff --git a/src/neural_networks/bound_tightening_serra_parallel.jl b/src/neural_networks/bounds.jl similarity index 100% rename from src/neural_networks/bound_tightening_serra_parallel.jl rename to src/neural_networks/bounds.jl diff --git a/src/neural_networks/JuMP_model.jl b/src/neural_networks/old/JuMP_model.jl similarity index 100% rename from src/neural_networks/JuMP_model.jl rename to src/neural_networks/old/JuMP_model.jl diff --git a/src/neural_networks/bound_tightening.jl b/src/neural_networks/old/bound_tightening.jl similarity index 100% rename from src/neural_networks/bound_tightening.jl rename to src/neural_networks/old/bound_tightening.jl From dc20ac7846d3405e177bb5d4baf21010dffa2dec Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Fri, 2 Feb 2024 14:23:46 +0200 Subject: [PATCH 16/32] new compression algorithm --- Project.toml | 1 + examples/compression_test.jl | 50 ++++++++++++++ src/Gogeta.jl | 3 + src/neural_networks/bounds.jl | 67 ++++++++++++++++++ src/neural_networks/compress.jl | 119 ++++++++++++++++++++++++++++++++ 5 files changed, 240 insertions(+) create mode 100644 examples/compression_test.jl create mode 100644 src/neural_networks/compress.jl diff --git a/Project.toml b/Project.toml index 7a6008b..7815f88 100644 --- a/Project.toml +++ b/Project.toml @@ -12,6 +12,7 @@ Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" GLPK = "60bf3e95-4087-53dc-ae20-288a0d20c6a6" Gurobi = "2e9cd046-0924-5485-92f1-d5272153d98b" JuMP = "4076af6c-e467-56ae-b986-b466b2749572" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" MLDatasets = "eb30cadb-4394-5ae3-aed4-317e484a6458" NPZ = "15e1cf62-19b3-5cfa-8e77-841668bca605" diff --git a/examples/compression_test.jl b/examples/compression_test.jl new file mode 100644 index 0000000..959fffe --- /dev/null +++ b/examples/compression_test.jl @@ -0,0 +1,50 @@ +data = rand(Float32, (2, 1000)) .- 0.5f0; +x_train = data[:, 1:750]; + +using Flux +using Random +using Revise +using Gogeta + +begin + Random.seed!(1234); + + model = Chain( + Dense(2 => 10, relu), + Dense(10 => 50, relu), + Dense(50 => 20, relu), + Dense(20 => 5, relu), + Dense(5 => 1) + ) +end + +begin + Random.seed!(1234); + + model = Chain( + Dense(2 => 10, relu), + Dense(10 => 20, relu), + Dense(20 => 5, relu), + Dense(5 => 1) + ) +end + +begin + Random.seed!(1234); + + model = Chain( + Dense(2 => 3, relu), + Dense(3 => 1) + ) +end + +solver_params = SolverParams(silent=true, threads=0, relax=false, time_limit=0) +@time jump, U, L, removed = compress(model, [1.0, 1.0], [-1.0, -1.0], solver_params); +vec(model(x_train)) ≈ [forward_pass!(jump, x_train[:, i])[1] for i in 1:750] + +@time _, U_full, L_full = NN_to_MIP(model, [1.0, 1.0], [-1.0, -1.0], solver_params; tighten_bounds=true); + +using Plots + +plot(collect(Iterators.flatten(U[1:end-1]))) +plot!(collect(Iterators.flatten(U_full[1:end-1]))) diff --git a/src/Gogeta.jl b/src/Gogeta.jl index 2b1daf2..ae03b9c 100644 --- a/src/Gogeta.jl +++ b/src/Gogeta.jl @@ -5,6 +5,9 @@ export NN_to_MIP, forward_pass!, SolverParams include("neural_networks/bounds.jl") +include("neural_networks/compress.jl") +export compress + include("tree_ensembles/types.jl") export TEModel, extract_evotrees_info diff --git a/src/neural_networks/bounds.jl b/src/neural_networks/bounds.jl index 64690aa..463d55a 100644 --- a/src/neural_networks/bounds.jl +++ b/src/neural_networks/bounds.jl @@ -47,4 +47,71 @@ function calculate_bounds(model::JuMP.Model, layer, neuron, W, b, neurons) println("Neuron: $neuron") return upper_bound, lower_bound +end + +function calculate_bounds_fast(model::JuMP.Model, layer, neuron, W, b, neurons) + + upper_exists::Bool = true + lower_exists::Bool = true + + function bounds_callback(cb_data, cb_where::Cint) + + # Only run at integer solutions + if cb_where == GRB_CB_MIPSOL + + objbound = Ref{Cdouble}() + objval = Ref{Cdouble}() + GRBcbget(cb_data, cb_where, GRB_CB_MIPSOL_OBJBND, objbound) + GRBcbget(cb_data, cb_where, GRB_CB_MIPSOL_OBJ, objval) + + if objective_sense(model) == MAX_SENSE + + if objval[] > 0 + upper_exists = true + GRBterminate(backend(model)) + end + + if objbound[] <= 0 + upper_exists = false + GRBterminate(backend(model)) + end + + elseif objective_sense(model) == MIN_SENSE + + if objval[] < 0 + lower_exists = true + GRBterminate(backend(model)) + end + + if objbound[] >= 0 + lower_exists = false + GRBterminate(backend(model)) + end + end + end + + end + + @objective(model, Max, b[layer][neuron] + sum(W[layer][neuron, i] * model[:x][layer-1, i] for i in neurons(layer-1))) + + set_attribute(model, "LazyConstraints", 1) + set_attribute(model, Gurobi.CallbackFunction(), bounds_callback) + + optimize!(model) + if objective_value(model) <= 0 upper_exists = false end + + set_objective_sense(model, MIN_SENSE) + optimize!(model) + if objective_value(model) >= 0 lower_exists = false end + + status = if upper_exists == false + "stabily inactive" + elseif lower_exists == false + "stabily active" + else + "normal" + end + println("Neuron: $neuron, $status") + + return upper_exists, lower_exists end \ No newline at end of file diff --git a/src/neural_networks/compress.jl b/src/neural_networks/compress.jl new file mode 100644 index 0000000..a9bd859 --- /dev/null +++ b/src/neural_networks/compress.jl @@ -0,0 +1,119 @@ +using Flux +using JuMP +using LinearAlgebra: rank, dot + +function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}, params; big_M=1000.0) + + K = length(model) + + @assert all([model[i].σ == relu for i in 1:K-1]) "Neural network must use the relu activation function." + @assert model[K].σ == identity "Neural network must use the identity function for the output layer." + + W = [Flux.params(model)[2*k-1] for k in 1:K] # W[i] = weight matrix for i:th layer + b = [Flux.params(model)[2*k] for k in 1:K] + + input_length = Int((length(W[1]) / length(b[1]))) + neuron_count = [length(b[k]) for k in eachindex(b)] + neurons(layer) = layer == 0 ? [i for i in 1:input_length] : [i for i in 1:neuron_count[layer]] + + @assert input_length == length(init_ub) == length(init_lb) "Initial bounds arrays must be the same length as the input layer" + + # build JuMP model + jump_model = direct_model(Gurobi.Optimizer()) + params.silent && set_silent(jump_model) + params.threads != 0 && set_attribute(jump_model, "Threads", params.threads) + params.relax && relax_integrality(jump_model) + params.time_limit != 0 && set_attribute(jump_model, "TimeLimit", params.time_limit) + + @variable(jump_model, x[layer = 0:K, neurons(layer)]) + @variable(jump_model, s[layer = 0:K-1, neurons(layer)]) + @variable(jump_model, z[layer = 0:K-1, neurons(layer)]) + + @constraint(jump_model, [j = 1:input_length], x[0, j] <= init_ub[j]) + @constraint(jump_model, [j = 1:input_length], x[0, j] >= init_lb[j]) + + bounds_U = Vector{Vector}(undef, K) + bounds_L = Vector{Vector}(undef, K) + + removed_neurons = Vector{Vector}(undef, K-1) + + for layer in 1:K-1 # hidden layers + + println("LAYER: $layer") + + bounds_U[layer] = fill(big_M, length(neurons(layer))) + bounds_L[layer] = fill(-big_M, length(neurons(layer))) + + bounds = map(neuron -> calculate_bounds_fast(jump_model, layer, neuron, W, b, neurons), neurons(layer)) + + stable_units = Set{Int}() # indices of stable neurons + unstable_units = false + + removed_neurons[layer] = Vector{Int}() + + for neuron in neurons(layer) + + if bounds[neuron][1] == false || iszero(W[layer][neuron, :]) # constant output + + if length(stable_units) > 0 || unstable_units == true || neuron < neuron_count[layer] + + if iszero(W[layer][neuron, :]) && b[layer][neuron] > 0 + + for neuron_next in neurons(layer+1) + b[layer+1][neuron_next] += W[layer+1][neuron_next, neuron] * b[layer][neuron] + end + + end + + W[layer] = W[layer][setdiff(neurons(layer), neuron), :] + b[layer] = b[layer][setdiff(neurons(layer), neuron)] + push!(removed_neurons[layer], neuron) + end + + elseif bounds[neuron][2] == false # stabily active + + if rank(W[layer][collect(union(stable_units, neuron))]) > length(stable_units) + push!(stable_units, neuron) + else + alpha = W[layer][collect(stable_units), :]' \ W[layer][neuron, :] + @assert dot(alpha, W[layer][collect(stable_units), :]) == W[layer][neuron, :] "Alpha calculation not working." + + for neuron_next in neurons(layer+1) + W[layer+1][neuron_next, collect(stable_units)] .+= sum(W[layer+1][neuron_next, neuron] * alpha) + b[layer+1][neuron_next] += W[layer+1][neuron_next, neuron] * (b[layer][neuron] + dot(b[layer][collect(stable_units)], alpha)) + end + + W[layer] = W[layer][setdiff(neurons(layer), neuron), :] + b[layer] = b[layer][setdiff(neurons(layer), neuron)] + push!(removed_neurons[layer], neuron) + end + else + unstable_units = true + end + + end + + # TODO neurons() function has to be changed as the layers are pruned + + if unstable_units == false # all units in the layer are stable + println("Fully stable layer") + end + + for neuron in 1:(neuron_count[layer] - length(removed_neurons[layer])) + @constraint(jump_model, x[layer, neuron] >= 0) + @constraint(jump_model, s[layer, neuron] >= 0) + set_binary(z[layer, neuron]) + + @constraint(jump_model, x[layer, neuron] <= bounds_U[layer][neuron] * (1 - z[layer, neuron])) + @constraint(jump_model, s[layer, neuron] <= -bounds_L[layer][neuron] * z[layer, neuron]) + + @constraint(jump_model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1, i] for i in neurons(layer-1))) + end + + end + + @constraint(jump_model, [neuron in 1:neuron_count[K]], x[K, neuron] == b[K][neuron] + sum(W[K][neuron, i] * x[K-1, i] for i in neurons(K-1))) + + return jump_model, bounds_U, bounds_L, removed_neurons + +end \ No newline at end of file From 763143e9b5bff345557118d3a8382308433565e8 Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Fri, 2 Feb 2024 15:43:33 +0200 Subject: [PATCH 17/32] changes to compression --- examples/compression_test.jl | 3 ++- src/neural_networks/compress.jl | 31 +++++++++++++------------------ 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/examples/compression_test.jl b/examples/compression_test.jl index 959fffe..ace01ea 100644 --- a/examples/compression_test.jl +++ b/examples/compression_test.jl @@ -39,7 +39,8 @@ begin end solver_params = SolverParams(silent=true, threads=0, relax=false, time_limit=0) -@time jump, U, L, removed = compress(model, [1.0, 1.0], [-1.0, -1.0], solver_params); + +@time jump, removed = compress(model, [1.0, 1.0], [-1.0, -1.0], solver_params); vec(model(x_train)) ≈ [forward_pass!(jump, x_train[:, i])[1] for i in 1:750] @time _, U_full, L_full = NN_to_MIP(model, [1.0, 1.0], [-1.0, -1.0], solver_params; tighten_bounds=true); diff --git a/src/neural_networks/compress.jl b/src/neural_networks/compress.jl index a9bd859..8809515 100644 --- a/src/neural_networks/compress.jl +++ b/src/neural_networks/compress.jl @@ -14,7 +14,7 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F input_length = Int((length(W[1]) / length(b[1]))) neuron_count = [length(b[k]) for k in eachindex(b)] - neurons(layer) = layer == 0 ? [i for i in 1:input_length] : [i for i in 1:neuron_count[layer]] + neurons(layer) = layer == 0 ? [i for i in 1:input_length] : [i for i in 1:length(b[layer])] @assert input_length == length(init_ub) == length(init_lb) "Initial bounds arrays must be the same length as the input layer" @@ -31,18 +31,12 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F @constraint(jump_model, [j = 1:input_length], x[0, j] <= init_ub[j]) @constraint(jump_model, [j = 1:input_length], x[0, j] >= init_lb[j]) - - bounds_U = Vector{Vector}(undef, K) - bounds_L = Vector{Vector}(undef, K) removed_neurons = Vector{Vector}(undef, K-1) for layer in 1:K-1 # hidden layers - println("LAYER: $layer") - - bounds_U[layer] = fill(big_M, length(neurons(layer))) - bounds_L[layer] = fill(-big_M, length(neurons(layer))) + println("\nLAYER: $layer") bounds = map(neuron -> calculate_bounds_fast(jump_model, layer, neuron, W, b, neurons), neurons(layer)) @@ -65,8 +59,10 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F end - W[layer] = W[layer][setdiff(neurons(layer), neuron), :] - b[layer] = b[layer][setdiff(neurons(layer), neuron)] + # TODO neurons() function has to be changed as the layers are pruned + + # W[layer] = W[layer][setdiff(neurons(layer), neuron), :] + # b[layer] = b[layer][setdiff(neurons(layer), neuron)] push!(removed_neurons[layer], neuron) end @@ -76,15 +72,15 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F push!(stable_units, neuron) else alpha = W[layer][collect(stable_units), :]' \ W[layer][neuron, :] - @assert dot(alpha, W[layer][collect(stable_units), :]) == W[layer][neuron, :] "Alpha calculation not working." + @assert dot(alpha, W[layer][collect(stable_units), :]') == W[layer][neuron, :] "Alpha calculation not working." for neuron_next in neurons(layer+1) W[layer+1][neuron_next, collect(stable_units)] .+= sum(W[layer+1][neuron_next, neuron] * alpha) b[layer+1][neuron_next] += W[layer+1][neuron_next, neuron] * (b[layer][neuron] + dot(b[layer][collect(stable_units)], alpha)) end - W[layer] = W[layer][setdiff(neurons(layer), neuron), :] - b[layer] = b[layer][setdiff(neurons(layer), neuron)] + # W[layer] = W[layer][setdiff(neurons(layer), neuron), :] + # b[layer] = b[layer][setdiff(neurons(layer), neuron)] push!(removed_neurons[layer], neuron) end else @@ -93,10 +89,9 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F end - # TODO neurons() function has to be changed as the layers are pruned - if unstable_units == false # all units in the layer are stable println("Fully stable layer") + # TODO implement folding code end for neuron in 1:(neuron_count[layer] - length(removed_neurons[layer])) @@ -104,8 +99,8 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F @constraint(jump_model, s[layer, neuron] >= 0) set_binary(z[layer, neuron]) - @constraint(jump_model, x[layer, neuron] <= bounds_U[layer][neuron] * (1 - z[layer, neuron])) - @constraint(jump_model, s[layer, neuron] <= -bounds_L[layer][neuron] * z[layer, neuron]) + @constraint(jump_model, x[layer, neuron] <= big_M * (1 - z[layer, neuron])) + @constraint(jump_model, s[layer, neuron] <= big_M * z[layer, neuron]) @constraint(jump_model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1, i] for i in neurons(layer-1))) end @@ -114,6 +109,6 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F @constraint(jump_model, [neuron in 1:neuron_count[K]], x[K, neuron] == b[K][neuron] + sum(W[K][neuron, i] * x[K-1, i] for i in neurons(K-1))) - return jump_model, bounds_U, bounds_L, removed_neurons + return jump_model, removed_neurons end \ No newline at end of file From 12d68df851af6cd6d53cfc8f9cd961b97fa9a241 Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Mon, 5 Feb 2024 14:25:48 +0200 Subject: [PATCH 18/32] new compression almost finished --- examples/compression_test.jl | 8 +-- src/neural_networks/bounds.jl | 17 ++----- src/neural_networks/compress.jl | 87 +++++++++++++++++++++++---------- 3 files changed, 71 insertions(+), 41 deletions(-) diff --git a/examples/compression_test.jl b/examples/compression_test.jl index ace01ea..f096e67 100644 --- a/examples/compression_test.jl +++ b/examples/compression_test.jl @@ -40,12 +40,14 @@ end solver_params = SolverParams(silent=true, threads=0, relax=false, time_limit=0) -@time jump, removed = compress(model, [1.0, 1.0], [-1.0, -1.0], solver_params); +@time jump, removed, compressed, U_comp, L_comp = compress(model, [1.0, 1.0], [-1.0, -1.0], solver_params); + vec(model(x_train)) ≈ [forward_pass!(jump, x_train[:, i])[1] for i in 1:750] +compressed(x_train) ≈ model(x_train) @time _, U_full, L_full = NN_to_MIP(model, [1.0, 1.0], [-1.0, -1.0], solver_params; tighten_bounds=true); using Plots -plot(collect(Iterators.flatten(U[1:end-1]))) -plot!(collect(Iterators.flatten(U_full[1:end-1]))) +plot(collect(Iterators.flatten(U_comp[1:end-1]))) +plot!(collect(Iterators.flatten(U_full[1:end-1]))) \ No newline at end of file diff --git a/src/neural_networks/bounds.jl b/src/neural_networks/bounds.jl index 463d55a..443bb6f 100644 --- a/src/neural_networks/bounds.jl +++ b/src/neural_networks/bounds.jl @@ -51,9 +51,6 @@ end function calculate_bounds_fast(model::JuMP.Model, layer, neuron, W, b, neurons) - upper_exists::Bool = true - lower_exists::Bool = true - function bounds_callback(cb_data, cb_where::Cint) # Only run at integer solutions @@ -67,24 +64,20 @@ function calculate_bounds_fast(model::JuMP.Model, layer, neuron, W, b, neurons) if objective_sense(model) == MAX_SENSE if objval[] > 0 - upper_exists = true GRBterminate(backend(model)) end if objbound[] <= 0 - upper_exists = false GRBterminate(backend(model)) end elseif objective_sense(model) == MIN_SENSE if objval[] < 0 - lower_exists = true GRBterminate(backend(model)) end if objbound[] >= 0 - lower_exists = false GRBterminate(backend(model)) end end @@ -98,20 +91,20 @@ function calculate_bounds_fast(model::JuMP.Model, layer, neuron, W, b, neurons) set_attribute(model, Gurobi.CallbackFunction(), bounds_callback) optimize!(model) - if objective_value(model) <= 0 upper_exists = false end + upper = objective_bound(model) set_objective_sense(model, MIN_SENSE) optimize!(model) - if objective_value(model) >= 0 lower_exists = false end + lower = objective_bound(model) - status = if upper_exists == false + status = if upper <= 0 "stabily inactive" - elseif lower_exists == false + elseif lower >= 0 "stabily active" else "normal" end println("Neuron: $neuron, $status") - return upper_exists, lower_exists + return upper, lower end \ No newline at end of file diff --git a/src/neural_networks/compress.jl b/src/neural_networks/compress.jl index 8809515..5f03a4e 100644 --- a/src/neural_networks/compress.jl +++ b/src/neural_networks/compress.jl @@ -9,12 +9,15 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F @assert all([model[i].σ == relu for i in 1:K-1]) "Neural network must use the relu activation function." @assert model[K].σ == identity "Neural network must use the identity function for the output layer." - W = [Flux.params(model)[2*k-1] for k in 1:K] # W[i] = weight matrix for i:th layer - b = [Flux.params(model)[2*k] for k in 1:K] + W = deepcopy([Flux.params(model)[2*k-1] for k in 1:K]) # W[i] = weight matrix for i:th layer + b = deepcopy([Flux.params(model)[2*k] for k in 1:K]) + + removed_neurons = Vector{Vector}(undef, K) + [removed_neurons[layer] = Vector{Int}() for layer in 1:K] input_length = Int((length(W[1]) / length(b[1]))) neuron_count = [length(b[k]) for k in eachindex(b)] - neurons(layer) = layer == 0 ? [i for i in 1:input_length] : [i for i in 1:length(b[layer])] + neurons(layer) = layer == 0 ? [i for i in 1:input_length] : [i for i in setdiff(1:length(b[layer]), removed_neurons[layer])] @assert input_length == length(init_ub) == length(init_lb) "Initial bounds arrays must be the same length as the input layer" @@ -32,24 +35,30 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F @constraint(jump_model, [j = 1:input_length], x[0, j] <= init_ub[j]) @constraint(jump_model, [j = 1:input_length], x[0, j] >= init_lb[j]) - removed_neurons = Vector{Vector}(undef, K-1) + bounds_U = Vector{Vector}(undef, K) + bounds_L = Vector{Vector}(undef, K) for layer in 1:K-1 # hidden layers println("\nLAYER: $layer") + bounds_U[layer] = fill(big_M, length(neurons(layer))) + bounds_L[layer] = fill(big_M, length(neurons(layer))) + bounds = map(neuron -> calculate_bounds_fast(jump_model, layer, neuron, W, b, neurons), neurons(layer)) + bounds_U[layer], bounds_L[layer] = [bound[1] > big_M ? big_M : bound[1] for bound in bounds], [bound[2] > big_M ? big_M : bound[2] for bound in bounds] + println() stable_units = Set{Int}() # indices of stable neurons unstable_units = false - - removed_neurons[layer] = Vector{Int}() for neuron in neurons(layer) - if bounds[neuron][1] == false || iszero(W[layer][neuron, :]) # constant output + if bounds_U[layer][neuron] <= 0 || iszero(W[layer][neuron, :]) # stabily inactive + + println("Neuron $neuron is stabily inactive") - if length(stable_units) > 0 || unstable_units == true || neuron < neuron_count[layer] + if neuron < neuron_count[layer] || length(stable_units) > 0 || unstable_units == true if iszero(W[layer][neuron, :]) && b[layer][neuron] > 0 @@ -59,28 +68,27 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F end - # TODO neurons() function has to be changed as the layers are pruned - - # W[layer] = W[layer][setdiff(neurons(layer), neuron), :] - # b[layer] = b[layer][setdiff(neurons(layer), neuron)] push!(removed_neurons[layer], neuron) end - elseif bounds[neuron][2] == false # stabily active + elseif bounds_L[layer][neuron] >= 0 # stabily active + + println("Neuron $neuron is stabily active") - if rank(W[layer][collect(union(stable_units, neuron))]) > length(stable_units) + if rank(W[layer][collect(union(stable_units, neuron)), :]) > length(stable_units) push!(stable_units, neuron) else - alpha = W[layer][collect(stable_units), :]' \ W[layer][neuron, :] - @assert dot(alpha, W[layer][collect(stable_units), :]') == W[layer][neuron, :] "Alpha calculation not working." + S = sort!(collect(stable_units)) + + alpha = transpose(W[layer][S, :]) \ W[layer][neuron, :] + + @assert transpose(W[layer][S, :]) * alpha ≈ W[layer][neuron, :] "Alpha calculation not working." for neuron_next in neurons(layer+1) - W[layer+1][neuron_next, collect(stable_units)] .+= sum(W[layer+1][neuron_next, neuron] * alpha) - b[layer+1][neuron_next] += W[layer+1][neuron_next, neuron] * (b[layer][neuron] + dot(b[layer][collect(stable_units)], alpha)) + W[layer+1][neuron_next, S] .+= sum(W[layer+1][neuron_next, neuron] * alpha) + b[layer+1][neuron_next] += W[layer+1][neuron_next, neuron] * (b[layer][neuron] + dot(b[layer][S], alpha)) end - # W[layer] = W[layer][setdiff(neurons(layer), neuron), :] - # b[layer] = b[layer][setdiff(neurons(layer), neuron)] push!(removed_neurons[layer], neuron) end else @@ -89,26 +97,53 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F end + println() + if unstable_units == false # all units in the layer are stable println("Fully stable layer") - # TODO implement folding code + + if length(stable_units) > 0 + # TODO implement folding code + else + output = model((init_ub + init_lb) ./ 2) + println("WHOLE NETWORK IS CONSTANT WITH OUTPUT: $output") + return + end end - for neuron in 1:(neuron_count[layer] - length(removed_neurons[layer])) + println("Removed $(length(removed_neurons[layer])) neurons") + + for neuron in neurons(layer) @constraint(jump_model, x[layer, neuron] >= 0) @constraint(jump_model, s[layer, neuron] >= 0) set_binary(z[layer, neuron]) - @constraint(jump_model, x[layer, neuron] <= big_M * (1 - z[layer, neuron])) - @constraint(jump_model, s[layer, neuron] <= big_M * z[layer, neuron]) + @constraint(jump_model, x[layer, neuron] <= bounds_U[layer][neuron] * (1 - z[layer, neuron])) + @constraint(jump_model, s[layer, neuron] <= -bounds_L[layer][neuron] * z[layer, neuron]) @constraint(jump_model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1, i] for i in neurons(layer-1))) end end - @constraint(jump_model, [neuron in 1:neuron_count[K]], x[K, neuron] == b[K][neuron] + sum(W[K][neuron, i] * x[K-1, i] for i in neurons(K-1))) + # output layer + @constraint(jump_model, [neuron in neurons(K)], x[K, neuron] == b[K][neuron] + sum(W[K][neuron, i] * x[K-1, i] for i in neurons(K-1))) + + # build compressed model + new_layers = []; + for layer in 1:K + if layer != K + W[layer] = W[layer][setdiff(1:size(W[layer])[1], removed_neurons[layer]), setdiff(1:size(W[layer])[2], layer == 1 ? [] : removed_neurons[layer-1])] + b[layer] = b[layer][setdiff(1:length(b[layer]), removed_neurons[layer])] + + push!(new_layers, Dense(W[layer], b[layer], relu)) + else + push!(new_layers, Dense(W[layer], b[layer])) + end + end + + new_model = Flux.Chain(new_layers...) - return jump_model, removed_neurons + return jump_model, removed_neurons, new_model, bounds_U, bounds_L end \ No newline at end of file From 9426965ebb32780dee5efde48a1d5018a7e1db59 Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Tue, 6 Feb 2024 14:38:35 +0200 Subject: [PATCH 19/32] finished compression algorithm --- src/neural_networks/bounds.jl | 4 ++-- src/neural_networks/compress.jl | 38 +++++++++++++++++++++++++++------ 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/src/neural_networks/bounds.jl b/src/neural_networks/bounds.jl index 443bb6f..11ed8c1 100644 --- a/src/neural_networks/bounds.jl +++ b/src/neural_networks/bounds.jl @@ -49,7 +49,7 @@ function calculate_bounds(model::JuMP.Model, layer, neuron, W, b, neurons) return upper_bound, lower_bound end -function calculate_bounds_fast(model::JuMP.Model, layer, neuron, W, b, neurons) +function calculate_bounds_fast(model::JuMP.Model, layer, neuron, W, b, neurons, layer_removed) function bounds_callback(cb_data, cb_where::Cint) @@ -85,7 +85,7 @@ function calculate_bounds_fast(model::JuMP.Model, layer, neuron, W, b, neurons) end - @objective(model, Max, b[layer][neuron] + sum(W[layer][neuron, i] * model[:x][layer-1, i] for i in neurons(layer-1))) + @objective(model, Max, b[layer][neuron] + sum(W[layer][neuron, i] * model[:x][layer-1-layer_removed, i] for i in neurons(layer-1-layer_removed))) set_attribute(model, "LazyConstraints", 1) set_attribute(model, Gurobi.CallbackFunction(), bounds_callback) diff --git a/src/neural_networks/compress.jl b/src/neural_networks/compress.jl index 5f03a4e..9dac440 100644 --- a/src/neural_networks/compress.jl +++ b/src/neural_networks/compress.jl @@ -38,6 +38,8 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F bounds_U = Vector{Vector}(undef, K) bounds_L = Vector{Vector}(undef, K) + layer_removed = false + for layer in 1:K-1 # hidden layers println("\nLAYER: $layer") @@ -45,7 +47,7 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F bounds_U[layer] = fill(big_M, length(neurons(layer))) bounds_L[layer] = fill(big_M, length(neurons(layer))) - bounds = map(neuron -> calculate_bounds_fast(jump_model, layer, neuron, W, b, neurons), neurons(layer)) + bounds = map(neuron -> calculate_bounds_fast(jump_model, layer, neuron, W, b, neurons, layer_removed), neurons(layer)) bounds_U[layer], bounds_L[layer] = [bound[1] > big_M ? big_M : bound[1] for bound in bounds], [bound[2] > big_M ? big_M : bound[2] for bound in bounds] println() @@ -103,7 +105,24 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F println("Fully stable layer") if length(stable_units) > 0 - # TODO implement folding code + + W_bar = Matrix{eltype(W[1][1])}(undef, neuron_count[layer-1], neuron_count[layer+1]) + b_bar = Vector{eltype(b[1][1])}(undef, neuron_count[layer+1]) + + for neuron_next in neurons(layer+1) + + b_bar[neuron_next] = b[layer+1][neuron_next] + sum([W[layer+1][neuron_next, k] * b[layer][k] for k in collect(stable_units)]) + + for neuron_previous in neurons(layer-1) + W_bar[neuron_next, neuron_previous] = sum([W[layer+1][neuron_next, k] * W[layer][k, neuron_previous] for k in collect(stable_units)]) + end + end + + W[layer+1] = W_bar + b[layer+1] = b_bar + + layer_removed = true + push!(removed_neurons[layer], neurons(layer)...) else output = model((init_ub + init_lb) ./ 2) println("WHOLE NETWORK IS CONSTANT WITH OUTPUT: $output") @@ -111,7 +130,7 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F end end - println("Removed $(length(removed_neurons[layer])) neurons") + println("Removed $(length(removed_neurons[layer]))/$(neuron_count[layer]) neurons") for neuron in neurons(layer) @constraint(jump_model, x[layer, neuron] >= 0) @@ -121,18 +140,23 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F @constraint(jump_model, x[layer, neuron] <= bounds_U[layer][neuron] * (1 - z[layer, neuron])) @constraint(jump_model, s[layer, neuron] <= -bounds_L[layer][neuron] * z[layer, neuron]) - @constraint(jump_model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1, i] for i in neurons(layer-1))) + @constraint(jump_model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1-layer_removed, i] for i in neurons(layer-1-layer_removed))) + end + + if length(neurons(layer)) > 0 + layer_removed = false end end # output layer - @constraint(jump_model, [neuron in neurons(K)], x[K, neuron] == b[K][neuron] + sum(W[K][neuron, i] * x[K-1, i] for i in neurons(K-1))) + @constraint(jump_model, [neuron in neurons(K)], x[K, neuron] == b[K][neuron] + sum(W[K][neuron, i] * x[K-1-layer_removed, i] for i in neurons(K-1-layer_removed))) # build compressed model new_layers = []; - for layer in 1:K - if layer != K + layers = findall(count -> count > 0, [neurons(l) for l in 1:K]) # exclude layers with no neurons + for layer in layers + if layer != last(layers) W[layer] = W[layer][setdiff(1:size(W[layer])[1], removed_neurons[layer]), setdiff(1:size(W[layer])[2], layer == 1 ? [] : removed_neurons[layer-1])] b[layer] = b[layer][setdiff(1:length(b[layer]), removed_neurons[layer])] From 3f22ae72938a97684f34b467ff428e7d4811f550 Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Thu, 8 Feb 2024 10:10:36 +0200 Subject: [PATCH 20/32] compression finally working --- examples/compression_test.jl | 32 +++- src/neural_networks/bounds.jl | 4 +- src/neural_networks/compress.jl | 276 +++++++++++++++++++++++--------- 3 files changed, 236 insertions(+), 76 deletions(-) diff --git a/examples/compression_test.jl b/examples/compression_test.jl index f096e67..cda81c1 100644 --- a/examples/compression_test.jl +++ b/examples/compression_test.jl @@ -49,5 +49,35 @@ compressed(x_train) ≈ model(x_train) using Plots +x = LinRange(-1.5, -0.5, 100) +y = LinRange(-0.5, 0.5, 100) + +contourf(x, y, (x, y) -> model(hcat(x, y)')[], c=cgrad(:viridis), lw=0) +contourf(x, y, (x, y) -> compressed(hcat(x, y)')[], c=cgrad(:viridis), lw=0) + +contourf(x, y, (x, y) -> forward_pass!(jump_model, vec(hcat(x, y)) .|> Float32)[], c=cgrad(:viridis), lw=0) +contourf(x, y, (x, y) -> forward_pass!(jump, vec(hcat(x, y)) .|> Float32)[], c=cgrad(:viridis), lw=0) + plot(collect(Iterators.flatten(U_comp[1:end-1]))) -plot!(collect(Iterators.flatten(U_full[1:end-1]))) \ No newline at end of file +plot!(collect(Iterators.flatten(U_full[1:end-1]))) + +subdir_path = joinpath(parent_dir, subdirs[1]) +model = load_model(n_neurons, subdir_path) + +solver_params = SolverParams(silent=true, threads=0, relax=false, time_limit=0) +@time jump, removed, compressed, U_comp, L_comp = compress(model, [-0.5, 0.5], [-1.5, -0.5], solver_params; big_M=1_000_000.0); + +U_full, L_full = get_bounds(subdir_path) + +@time _, removed, compressed, U_comp, L_comp = compress(model, [-0.5, 0.5], [-1.5, -0.5], U_full, -L_full); +contourf(x, y, (x, y) -> compressed(hcat(x, y)')[], c=cgrad(:viridis), lw=0) + +x1 = rand(Float32, 10) .- 1.5 +x2 = rand(Float32, 10) .- 0.5 +x = hcat(x1, x2)' + +vec(model(x)) ≈ [forward_pass!(jump, x[:, i] .|> Float32) for i in 1:size(x)[2]] +vec(compressed(x)) ≈ [forward_pass!(jump, x[:, i] .|> Float32)[] for i in 1:size(x)[2]] +compressed(x) ≈ model(x) + +@time jump_model, U_full, L_full = NN_to_MIP(model, [-0.5, 0.5], [-1.5, -0.5], solver_params; tighten_bounds=true, big_M=100_000.0); \ No newline at end of file diff --git a/src/neural_networks/bounds.jl b/src/neural_networks/bounds.jl index 11ed8c1..f5cd7d4 100644 --- a/src/neural_networks/bounds.jl +++ b/src/neural_networks/bounds.jl @@ -49,7 +49,7 @@ function calculate_bounds(model::JuMP.Model, layer, neuron, W, b, neurons) return upper_bound, lower_bound end -function calculate_bounds_fast(model::JuMP.Model, layer, neuron, W, b, neurons, layer_removed) +function calculate_bounds_fast(model::JuMP.Model, layer, neuron, W, b, neurons, layers_removed) function bounds_callback(cb_data, cb_where::Cint) @@ -85,7 +85,7 @@ function calculate_bounds_fast(model::JuMP.Model, layer, neuron, W, b, neurons, end - @objective(model, Max, b[layer][neuron] + sum(W[layer][neuron, i] * model[:x][layer-1-layer_removed, i] for i in neurons(layer-1-layer_removed))) + @objective(model, Max, b[layer][neuron] + sum(W[layer][neuron, i] * model[:x][layer-1-layers_removed, i] for i in neurons(layer-1-layers_removed))) set_attribute(model, "LazyConstraints", 1) set_attribute(model, Gurobi.CallbackFunction(), bounds_callback) diff --git a/src/neural_networks/compress.jl b/src/neural_networks/compress.jl index 9dac440..19d8f02 100644 --- a/src/neural_networks/compress.jl +++ b/src/neural_networks/compress.jl @@ -2,7 +2,183 @@ using Flux using JuMP using LinearAlgebra: rank, dot -function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}, params; big_M=1000.0) +# function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}, params; big_M=1000.0) + +# K = length(model) + +# @assert all([model[i].σ == relu for i in 1:K-1]) "Neural network must use the relu activation function." +# @assert model[K].σ == identity "Neural network must use the identity function for the output layer." + +# W = deepcopy([Flux.params(model)[2*k-1] for k in 1:K]) # W[i] = weight matrix for i:th layer +# b = deepcopy([Flux.params(model)[2*k] for k in 1:K]) + +# removed_neurons = Vector{Vector}(undef, K) +# [removed_neurons[layer] = Vector{Int}() for layer in 1:K] + +# input_length = Int((length(W[1]) / length(b[1]))) +# neuron_count = [length(b[k]) for k in eachindex(b)] +# neurons(layer) = layer == 0 ? [i for i in 1:input_length] : [i for i in setdiff(1:length(b[layer]), removed_neurons[layer])] + +# @assert input_length == length(init_ub) == length(init_lb) "Initial bounds arrays must be the same length as the input layer" + +# # build JuMP model +# jump_model = direct_model(Gurobi.Optimizer()) +# params.silent && set_silent(jump_model) +# params.threads != 0 && set_attribute(jump_model, "Threads", params.threads) +# params.relax && relax_integrality(jump_model) +# params.time_limit != 0 && set_attribute(jump_model, "TimeLimit", params.time_limit) + +# @variable(jump_model, x[layer = 0:K, neurons(layer)]) +# @variable(jump_model, s[layer = 1:K-1, neurons(layer)]) +# @variable(jump_model, z[layer = 1:K-1, neurons(layer)]) + +# @constraint(jump_model, [j = 1:input_length], x[0, j] <= init_ub[j]) +# @constraint(jump_model, [j = 1:input_length], x[0, j] >= init_lb[j]) + +# bounds_U = Vector{Vector}(undef, K) +# bounds_L = Vector{Vector}(undef, K) + +# layers_removed = 0 + +# for layer in 1:K-1 # hidden layers + +# println("\nLAYER: $layer") + +# bounds_U[layer] = fill(big_M, length(neurons(layer))) +# bounds_L[layer] = fill(big_M, length(neurons(layer))) + +# bounds = map(neuron -> calculate_bounds_fast(jump_model, layer, neuron, W, b, neurons, layers_removed), neurons(layer)) +# bounds_U[layer], bounds_L[layer] = [bound[1] > big_M ? big_M : bound[1] for bound in bounds], [bound[2] > big_M ? big_M : bound[2] for bound in bounds] +# println() + +# stable_units = Set{Int}() # indices of stable neurons +# unstable_units = false + +# for neuron in neurons(layer) + +# if bounds_U[layer][neuron] <= 0 || iszero(W[layer][neuron, :]) # stabily inactive + +# #println("Neuron $neuron is stabily inactive") + +# if neuron < last(neurons(layer)) || length(stable_units) > 0 || unstable_units == true + +# if iszero(W[layer][neuron, :]) && b[layer][neuron] > 0 + +# for neuron_next in neurons(layer+1) +# b[layer+1][neuron_next] += W[layer+1][neuron_next, neuron] * b[layer][neuron] +# end + +# end + +# push!(removed_neurons[layer], neuron) +# end + +# elseif bounds_L[layer][neuron] >= 0 # stabily active + +# #println("Neuron $neuron is stabily active") + +# if rank(W[layer][collect(union(stable_units, neuron)), :]) > length(stable_units) +# push!(stable_units, neuron) +# else +# S = sort!(collect(stable_units)) + +# alpha = transpose(W[layer][S, :]) \ W[layer][neuron, :] + +# @assert transpose(W[layer][S, :]) * alpha ≈ W[layer][neuron, :] "Alpha calculation not working." + +# for neuron_next in neurons(layer+1) +# W[layer+1][neuron_next, S] .+= sum(W[layer+1][neuron_next, neuron] * alpha) +# b[layer+1][neuron_next] += W[layer+1][neuron_next, neuron] * (b[layer][neuron] + dot(b[layer][S], alpha)) +# end + +# push!(removed_neurons[layer], neuron) +# end +# else +# unstable_units = true +# end + +# end + +# println() + +# if unstable_units == false # all units in the layer are stable +# println("Fully stable layer") + +# if length(stable_units) > 0 + +# W_bar = Matrix{eltype(W[1][1])}(undef, neuron_count[layer+1], neuron_count[layer-1]) +# b_bar = Vector{eltype(b[1][1])}(undef, neuron_count[layer+1]) + +# for neuron_next in neurons(layer+1) + +# b_bar[neuron_next] = b[layer+1][neuron_next] + sum([W[layer+1][neuron_next, k] * b[layer][k] for k in collect(stable_units)]) + +# for neuron_previous in neurons(layer-1) +# W_bar[neuron_next, neuron_previous] = sum([W[layer+1][neuron_next, k] * W[layer][k, neuron_previous] for k in collect(stable_units)]) +# end +# end + +# W[layer+1] = W_bar +# b[layer+1] = b_bar + +# layers_removed += 1 +# push!(removed_neurons[layer], neurons(layer)...) +# else +# output = model((init_ub + init_lb) ./ 2) +# println("WHOLE NETWORK IS CONSTANT WITH OUTPUT: $output") +# return +# end +# end + +# println("Removed $(length(removed_neurons[layer]))/$(neuron_count[layer]) neurons") + +# for neuron in neurons(layer) +# @constraint(jump_model, x[layer, neuron] >= 0) +# @constraint(jump_model, s[layer, neuron] >= 0) +# set_binary(z[layer, neuron]) + +# @constraint(jump_model, x[layer, neuron] <= bounds_U[layer][neuron] * (1 - z[layer, neuron])) +# @constraint(jump_model, s[layer, neuron] <= -bounds_L[layer][neuron] * z[layer, neuron]) + +# @constraint(jump_model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1-layers_removed, i] for i in neurons(layer-1-layers_removed))) +# end + +# if length(neurons(layer)) > 0 +# layers_removed = 0 +# end + +# end + +# # output layer +# @constraint(jump_model, [neuron in neurons(K)], x[K, neuron] == b[K][neuron] + sum(W[K][neuron, i] * x[K-1-layers_removed, i] for i in neurons(K-1-layers_removed))) + +# println("Weights: $([size(W[layer]) for layer in 1:K])") + +# # build compressed model +# new_layers = []; +# layers = findall(neurons -> length(neurons) > 0, [neurons(l) for l in 1:K]) # exclude layers with no neurons +# for (i, layer) in enumerate(layers) + +# if layers[i-1] != layer-1 # previous layer was removed +# else +# W[layer] = W[layer][setdiff(1:size(W[layer])[1], removed_neurons[layer]), setdiff(1:size(W[layer])[2], layer == 1 ? [] : removed_neurons[layers[i-1]])] +# b[layer] = b[layer][setdiff(1:length(b[layer]), removed_neurons[layer])] +# end + +# if layer != last(layers) +# push!(new_layers, Dense(W[layer], b[layer], relu)) +# else +# push!(new_layers, Dense(W[layer], b[layer])) +# end +# end + +# new_model = Flux.Chain(new_layers...) + +# return jump_model, removed_neurons, new_model, bounds_U, bounds_L + +# end + +function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}, bounds_U, bounds_L) K = length(model) @@ -17,78 +193,42 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F input_length = Int((length(W[1]) / length(b[1]))) neuron_count = [length(b[k]) for k in eachindex(b)] - neurons(layer) = layer == 0 ? [i for i in 1:input_length] : [i for i in setdiff(1:length(b[layer]), removed_neurons[layer])] - - @assert input_length == length(init_ub) == length(init_lb) "Initial bounds arrays must be the same length as the input layer" + neurons(layer) = layer == 0 ? [i for i in 1:input_length] : [i for i in setdiff(1:neuron_count[layer], removed_neurons[layer])] - # build JuMP model - jump_model = direct_model(Gurobi.Optimizer()) - params.silent && set_silent(jump_model) - params.threads != 0 && set_attribute(jump_model, "Threads", params.threads) - params.relax && relax_integrality(jump_model) - params.time_limit != 0 && set_attribute(jump_model, "TimeLimit", params.time_limit) - - @variable(jump_model, x[layer = 0:K, neurons(layer)]) - @variable(jump_model, s[layer = 0:K-1, neurons(layer)]) - @variable(jump_model, z[layer = 0:K-1, neurons(layer)]) - - @constraint(jump_model, [j = 1:input_length], x[0, j] <= init_ub[j]) - @constraint(jump_model, [j = 1:input_length], x[0, j] >= init_lb[j]) - - bounds_U = Vector{Vector}(undef, K) - bounds_L = Vector{Vector}(undef, K) - - layer_removed = false + layers_removed = 0 # how many strictly preceding layers removed at current loop iteration for layer in 1:K-1 # hidden layers - println("\nLAYER: $layer") - - bounds_U[layer] = fill(big_M, length(neurons(layer))) - bounds_L[layer] = fill(big_M, length(neurons(layer))) - - bounds = map(neuron -> calculate_bounds_fast(jump_model, layer, neuron, W, b, neurons, layer_removed), neurons(layer)) - bounds_U[layer], bounds_L[layer] = [bound[1] > big_M ? big_M : bound[1] for bound in bounds], [bound[2] > big_M ? big_M : bound[2] for bound in bounds] - println() - stable_units = Set{Int}() # indices of stable neurons unstable_units = false - for neuron in neurons(layer) + for neuron in 1:neuron_count[layer] if bounds_U[layer][neuron] <= 0 || iszero(W[layer][neuron, :]) # stabily inactive - println("Neuron $neuron is stabily inactive") - if neuron < neuron_count[layer] || length(stable_units) > 0 || unstable_units == true if iszero(W[layer][neuron, :]) && b[layer][neuron] > 0 - - for neuron_next in neurons(layer+1) + for neuron_next in 1:neuron_count[layer+1] # adjust biases b[layer+1][neuron_next] += W[layer+1][neuron_next, neuron] * b[layer][neuron] end - end push!(removed_neurons[layer], neuron) end elseif bounds_L[layer][neuron] >= 0 # stabily active - - println("Neuron $neuron is stabily active") if rank(W[layer][collect(union(stable_units, neuron)), :]) > length(stable_units) push!(stable_units, neuron) else - S = sort!(collect(stable_units)) + S = collect(stable_units) alpha = transpose(W[layer][S, :]) \ W[layer][neuron, :] - @assert transpose(W[layer][S, :]) * alpha ≈ W[layer][neuron, :] "Alpha calculation not working." - - for neuron_next in neurons(layer+1) - W[layer+1][neuron_next, S] .+= sum(W[layer+1][neuron_next, neuron] * alpha) - b[layer+1][neuron_next] += W[layer+1][neuron_next, neuron] * (b[layer][neuron] + dot(b[layer][S], alpha)) + for neuron_next in 1:neuron_count[layer+1] # adjust weights and biases + W[layer+1][neuron_next, S] .+= W[layer+1][neuron_next, neuron] * alpha + b[layer+1][neuron_next] += W[layer+1][neuron_next, neuron] * (b[layer][neuron] - dot(b[layer][S], alpha)) end push!(removed_neurons[layer], neuron) @@ -99,29 +239,29 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F end - println() - if unstable_units == false # all units in the layer are stable println("Fully stable layer") if length(stable_units) > 0 - W_bar = Matrix{eltype(W[1][1])}(undef, neuron_count[layer-1], neuron_count[layer+1]) + W_bar = Matrix{eltype(W[1][1])}(undef, neuron_count[layer+1], neuron_count[layer-1-layers_removed]) b_bar = Vector{eltype(b[1][1])}(undef, neuron_count[layer+1]) - for neuron_next in neurons(layer+1) + S = collect(stable_units) + + for neuron_next in 1:neuron_count[layer+1] - b_bar[neuron_next] = b[layer+1][neuron_next] + sum([W[layer+1][neuron_next, k] * b[layer][k] for k in collect(stable_units)]) + b_bar[neuron_next] = b[layer+1][neuron_next] + dot(W[layer+1][neuron_next, S], b[layer][S]) - for neuron_previous in neurons(layer-1) - W_bar[neuron_next, neuron_previous] = sum([W[layer+1][neuron_next, k] * W[layer][k, neuron_previous] for k in collect(stable_units)]) + for neuron_previous in 1:neuron_count[layer-1-layers_removed] + W_bar[neuron_next, neuron_previous] = dot(W[layer+1][neuron_next, S], W[layer][S, neuron_previous]) end end W[layer+1] = W_bar b[layer+1] = b_bar - layer_removed = true + layers_removed += 1 push!(removed_neurons[layer], neurons(layer)...) else output = model((init_ub + init_lb) ./ 2) @@ -132,34 +272,24 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F println("Removed $(length(removed_neurons[layer]))/$(neuron_count[layer]) neurons") - for neuron in neurons(layer) - @constraint(jump_model, x[layer, neuron] >= 0) - @constraint(jump_model, s[layer, neuron] >= 0) - set_binary(z[layer, neuron]) - - @constraint(jump_model, x[layer, neuron] <= bounds_U[layer][neuron] * (1 - z[layer, neuron])) - @constraint(jump_model, s[layer, neuron] <= -bounds_L[layer][neuron] * z[layer, neuron]) - - @constraint(jump_model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1-layer_removed, i] for i in neurons(layer-1-layer_removed))) - end - if length(neurons(layer)) > 0 - layer_removed = false + layers_removed = 0 end end - # output layer - @constraint(jump_model, [neuron in neurons(K)], x[K, neuron] == b[K][neuron] + sum(W[K][neuron, i] * x[K-1-layer_removed, i] for i in neurons(K-1-layer_removed))) + println(length.([neurons(l) for l in 0:K])) + println([neurons(l) for l in 0:K]) # build compressed model new_layers = []; - layers = findall(count -> count > 0, [neurons(l) for l in 1:K]) # exclude layers with no neurons - for layer in layers - if layer != last(layers) - W[layer] = W[layer][setdiff(1:size(W[layer])[1], removed_neurons[layer]), setdiff(1:size(W[layer])[2], layer == 1 ? [] : removed_neurons[layer-1])] - b[layer] = b[layer][setdiff(1:length(b[layer]), removed_neurons[layer])] + layers = findall(neurons -> length(neurons) > 0, [neurons(l) for l in 1:K]) # layers with neurons + for (i, layer) in enumerate(layers) + W[layer] = W[layer][neurons(layer), neurons(i == 1 ? 0 : layers[i-1])] + b[layer] = b[layer][neurons(layer)] + + if layer != last(layers) push!(new_layers, Dense(W[layer], b[layer], relu)) else push!(new_layers, Dense(W[layer], b[layer])) @@ -168,6 +298,6 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F new_model = Flux.Chain(new_layers...) - return jump_model, removed_neurons, new_model, bounds_U, bounds_L + return nothing, removed_neurons, new_model, bounds_U, bounds_L end \ No newline at end of file From 9e6e9d1a2b9945bc68830d421b6f0fcb28688e15 Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Thu, 8 Feb 2024 11:47:19 +0200 Subject: [PATCH 21/32] vilhelm's demo working with new code --- examples/compression_test.jl | 29 +++++++++++++++++++++++++++-- src/neural_networks/compress.jl | 11 +++++------ 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/examples/compression_test.jl b/examples/compression_test.jl index cda81c1..85b0ab6 100644 --- a/examples/compression_test.jl +++ b/examples/compression_test.jl @@ -67,9 +67,34 @@ model = load_model(n_neurons, subdir_path) solver_params = SolverParams(silent=true, threads=0, relax=false, time_limit=0) @time jump, removed, compressed, U_comp, L_comp = compress(model, [-0.5, 0.5], [-1.5, -0.5], solver_params; big_M=1_000_000.0); -U_full, L_full = get_bounds(subdir_path) +U_data, L_data = get_bounds(subdir_path) +U_data = U_data[3:end] +L_data = L_data[3:end] -@time _, removed, compressed, U_comp, L_comp = compress(model, [-0.5, 0.5], [-1.5, -0.5], U_full, -L_full); +""" +""" + +b = [Flux.params(model)[2*k] for k in 1:length(model)] +neuron_count = [length(b[k]) for k in eachindex(b)] + +U_full = Vector{Vector}(undef, length(model)) +L_full = Vector{Vector}(undef, length(model)) + +[U_full[layer] = Vector{Float64}(undef, neuron_count[layer]) for layer in 1:length(model)] +[L_full[layer] = Vector{Float64}(undef, neuron_count[layer]) for layer in 1:length(model)] + +for layer in 1:length(model) + for neuron in 1:neuron_count[layer] + U_full[layer][neuron] = U_data[neuron + (layer == 1 ? 0 : cumsum(neuron_count)[layer-1])] + L_full[layer][neuron] = L_data[neuron + (layer == 1 ? 0 : cumsum(neuron_count)[layer-1])] + end +end + +collect(Iterators.flatten(U_full)) == U_data +collect(Iterators.flatten(L_full)) == L_data + +@time _, removed, compressed, U_comp, L_comp = compress(model, [-0.5, 0.5], [-1.5, -0.5], U_full, L_full); +contourf(x, y, (x, y) -> model(hcat(x, y)')[], c=cgrad(:viridis), lw=0) contourf(x, y, (x, y) -> compressed(hcat(x, y)')[], c=cgrad(:viridis), lw=0) x1 = rand(Float32, 10) .- 1.5 diff --git a/src/neural_networks/compress.jl b/src/neural_networks/compress.jl index 19d8f02..4eb18a8 100644 --- a/src/neural_networks/compress.jl +++ b/src/neural_networks/compress.jl @@ -195,10 +195,12 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F neuron_count = [length(b[k]) for k in eachindex(b)] neurons(layer) = layer == 0 ? [i for i in 1:input_length] : [i for i in setdiff(1:neuron_count[layer], removed_neurons[layer])] - layers_removed = 0 # how many strictly preceding layers removed at current loop iteration + layers_removed = 0 # how many strictly preceding layers have been removed at current loop iteration for layer in 1:K-1 # hidden layers + println("\nLAYER $layer") + stable_units = Set{Int}() # indices of stable neurons unstable_units = false @@ -266,7 +268,7 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F else output = model((init_ub + init_lb) ./ 2) println("WHOLE NETWORK IS CONSTANT WITH OUTPUT: $output") - return + return output end end @@ -274,13 +276,10 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F if length(neurons(layer)) > 0 layers_removed = 0 - end + end end - println(length.([neurons(l) for l in 0:K])) - println([neurons(l) for l in 0:K]) - # build compressed model new_layers = []; layers = findall(neurons -> length(neurons) > 0, [neurons(l) for l in 1:K]) # layers with neurons From aa934d56891471d61e2e5afa6a3b00202e15e9d8 Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Fri, 9 Feb 2024 10:35:25 +0200 Subject: [PATCH 22/32] working compression without callbacks --- examples/compression_test.jl | 19 ++- examples/nn_comp_test.jl | 55 +++++++ src/Gogeta.jl | 2 +- src/neural_networks/NN_to_MIP.jl | 16 +-- src/neural_networks/bounds.jl | 26 ++-- src/neural_networks/compress.jl | 240 +++++++++++++++---------------- 6 files changed, 211 insertions(+), 147 deletions(-) create mode 100644 examples/nn_comp_test.jl diff --git a/examples/compression_test.jl b/examples/compression_test.jl index 85b0ab6..8859721 100644 --- a/examples/compression_test.jl +++ b/examples/compression_test.jl @@ -94,8 +94,25 @@ collect(Iterators.flatten(U_full)) == U_data collect(Iterators.flatten(L_full)) == L_data @time _, removed, compressed, U_comp, L_comp = compress(model, [-0.5, 0.5], [-1.5, -0.5], U_full, L_full); + +solver_params = SolverParams(silent=true, threads=0, relax=false, time_limit=0) + +@time compression_results = compress_fast(model, [-0.5, 0.5], [-1.5, -0.5], solver_params); +jump_model, removed_neurons, compressed_model, bounds_U, bounds_L = compression_results; + +@time bound_results = NN_to_MIP(model, [-0.5, 0.5], [-1.5, -0.5], solver_params; tighten_bounds=true, big_M=100_000.0); + +bound_results[3] + +@time bound_compression = compress(model, [-0.5, 0.5], [-1.5, -0.5], bounds_U, bounds_L); + +@time bound_results = NN_to_MIP(compressed_model, [-0.5, 0.5], [-1.5, -0.5], solver_params; tighten_bounds=true, big_M=100_000.0); + +bound_results[1] + +contourf(x, y, (x, y) -> forward_pass!(jump_model, vec(hcat(x, y)) .|> Float32)[], c=cgrad(:viridis), lw=0) contourf(x, y, (x, y) -> model(hcat(x, y)')[], c=cgrad(:viridis), lw=0) -contourf(x, y, (x, y) -> compressed(hcat(x, y)')[], c=cgrad(:viridis), lw=0) +contourf(x, y, (x, y) -> compressed_model(hcat(x, y)')[], c=cgrad(:viridis), lw=0) x1 = rand(Float32, 10) .- 1.5 x2 = rand(Float32, 10) .- 0.5 diff --git a/examples/nn_comp_test.jl b/examples/nn_comp_test.jl new file mode 100644 index 0000000..f8d619b --- /dev/null +++ b/examples/nn_comp_test.jl @@ -0,0 +1,55 @@ +using Flux +using Random +using Revise +using Gogeta +using Test +using Plots + +begin + Random.seed!(1234); + + model = Chain( + Dense(2 => 10, relu), + Dense(10 => 50, relu), + Dense(50 => 20, relu), + Dense(20 => 5, relu), + Dense(5 => 1) + ) +end + +init_U = [-0.5, 0.5]; +init_L = [-1.0, -1.0]; + +x1 = (rand(100) * (init_U[1] - init_L[1])) .+ init_L[1]; +x2 = (rand(100) * (init_U[2] - init_L[2])) .+ init_L[2]; +x = transpose(hcat(x1, x2)) .|> Float32; + +solver_params = SolverParams(silent=true, threads=0, relax=false, time_limit=0); + +# compress with compress fast +@time compression_results = compress_fast(model, init_U, init_L, solver_params); +jump_model, removed_neurons, compressed_model, bounds_U, bounds_L = compression_results; + +# test that jump model and compressed model produce same results as original +@test vec(model(x)) ≈ [forward_pass!(jump_model, x[:, i])[] for i in 1:size(x)[2]] +@test compressed_model(x) ≈ model(x) + +# perform bound tightening +@time nn_jump, U_correct, L_correct = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds=true); + +# test that created jump model is equal to the original +@test vec(model(x)) ≈ [forward_pass!(nn_jump, x[:, i])[] for i in 1:size(x)[2]] + +# compare bounds with/without fast compression +plot(collect(Iterators.flatten(bounds_U[1:end-1]))) +plot!(collect(Iterators.flatten(U_correct[1:end-1]))) + +plot!(collect(Iterators.flatten(bounds_L[1:end-1]))) +plot!(collect(Iterators.flatten(L_correct[1:end-1]))) + +# contour plot the model +x_range = LinRange(init_L[1], init_U[1], 100); +y_range = LinRange(init_L[2], init_U[2], 100); + +contourf(x_range, y_range, (x, y) -> model(hcat(x, y)')[], c=cgrad(:viridis), lw=0) +contourf(x_range, y_range, (x, y) -> compressed_model(hcat(x, y)')[], c=cgrad(:viridis), lw=0) \ No newline at end of file diff --git a/src/Gogeta.jl b/src/Gogeta.jl index ae03b9c..5793825 100644 --- a/src/Gogeta.jl +++ b/src/Gogeta.jl @@ -6,7 +6,7 @@ export NN_to_MIP, forward_pass!, SolverParams include("neural_networks/bounds.jl") include("neural_networks/compress.jl") -export compress +export compress, compress_fast include("tree_ensembles/types.jl") export TEModel, extract_evotrees_info diff --git a/src/neural_networks/NN_to_MIP.jl b/src/neural_networks/NN_to_MIP.jl index a0c098b..be01b12 100644 --- a/src/neural_networks/NN_to_MIP.jl +++ b/src/neural_networks/NN_to_MIP.jl @@ -9,7 +9,7 @@ using Distributed time_limit::Float64 end -function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}, solver_params::SolverParams; tighten_bounds::Bool=false, big_M::Float64=1000.0) +function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}, solver_params::SolverParams; tighten_bounds::Bool=false) K = length(NN_model) # number of layers (input layer not included) @assert reduce(&, [NN_model[i].σ == relu for i in 1:K-1]) "Neural network must use the relu activation function." @@ -29,8 +29,8 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect set_solver_params!(jump_model, solver_params) @variable(jump_model, x[layer = 0:K, neurons(layer)]) - @variable(jump_model, s[layer = 0:K-1, neurons(layer)]) - @variable(jump_model, z[layer = 0:K-1, neurons(layer)]) + @variable(jump_model, s[layer = 1:K-1, neurons(layer)]) + @variable(jump_model, z[layer = 1:K-1, neurons(layer)]) @constraint(jump_model, [j = 1:input_length], x[0, j] <= init_ub[j]) @constraint(jump_model, [j = 1:input_length], x[0, j] >= init_lb[j]) @@ -40,8 +40,8 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect for layer in 1:K # hidden layers and output - bounds_U[layer] = fill(big_M, length(neurons(layer))) - bounds_L[layer] = fill(big_M, length(neurons(layer))) + bounds_U[layer] = Vector{Float64}(undef, length(neurons(layer))) + bounds_L[layer] = Vector{Float64}(undef, length(neurons(layer))) println("\nLAYER $layer") @@ -51,7 +51,7 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect else map(neuron -> calculate_bounds(jump_model, layer, neuron, W, b, neurons), neurons(layer)) end - bounds_U[layer], bounds_L[layer] = [bound[1] > big_M ? big_M : bound[1] for bound in bounds], [bound[2] > big_M ? big_M : bound[2] for bound in bounds] + bounds_U[layer], bounds_L[layer] = [bound[1] for bound in bounds], [bound[2] for bound in bounds] end if layer == K # output bounds calculated but no unnecessary constraints added @@ -65,7 +65,7 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect set_binary(z[layer, neuron]) @constraint(jump_model, x[layer, neuron] <= bounds_U[layer][neuron] * (1 - z[layer, neuron])) - @constraint(jump_model, s[layer, neuron] <= bounds_L[layer][neuron] * z[layer, neuron]) + @constraint(jump_model, s[layer, neuron] <= -bounds_L[layer][neuron] * z[layer, neuron]) @constraint(jump_model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1, i] for i in neurons(layer-1))) @@ -78,7 +78,7 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect return jump_model, bounds_U, bounds_L end -function forward_pass!(jump_model::JuMP.Model, input::Vector{Float32}) +function forward_pass!(jump_model::JuMP.Model, input) @assert length(input) == length(jump_model[:x][0, :]) "Incorrect input length." [fix(jump_model[:x][0, i], input[i], force=true) for i in eachindex(input)] diff --git a/src/neural_networks/bounds.jl b/src/neural_networks/bounds.jl index f5cd7d4..22c7eed 100644 --- a/src/neural_networks/bounds.jl +++ b/src/neural_networks/bounds.jl @@ -39,9 +39,9 @@ function calculate_bounds(model::JuMP.Model, layer, neuron, W, b, neurons) optimize!(model) lower_bound = if termination_status(model) == OPTIMAL - max(-objective_value(model), 0.0) + min(objective_value(model), 0.0) else - max(-objective_bound(model), 0.0) + min(objective_bound(model), 0.0) end println("Neuron: $neuron") @@ -63,21 +63,29 @@ function calculate_bounds_fast(model::JuMP.Model, layer, neuron, W, b, neurons, if objective_sense(model) == MAX_SENSE + println("UPPER: Bound: $(objbound[]), objective: $(objval[])") + if objval[] > 0 + println("UPPER positive: $(objbound[])") GRBterminate(backend(model)) end - + if objbound[] <= 0 + println("UPPER negative: $(objbound[])") GRBterminate(backend(model)) end elseif objective_sense(model) == MIN_SENSE + println("LOWER: Bound: $(objbound[]), objective: $(objval[])") + if objval[] < 0 + println("LOWER negative: $(objbound[])") GRBterminate(backend(model)) end - + if objbound[] >= 0 + println("LOWER positive: $(objbound[])") GRBterminate(backend(model)) end end @@ -87,15 +95,15 @@ function calculate_bounds_fast(model::JuMP.Model, layer, neuron, W, b, neurons, @objective(model, Max, b[layer][neuron] + sum(W[layer][neuron, i] * model[:x][layer-1-layers_removed, i] for i in neurons(layer-1-layers_removed))) - set_attribute(model, "LazyConstraints", 1) - set_attribute(model, Gurobi.CallbackFunction(), bounds_callback) + #set_attribute(model, "LazyConstraints", 1) + #set_attribute(model, Gurobi.CallbackFunction(), bounds_callback) optimize!(model) - upper = objective_bound(model) + upper = max(objective_bound(model), 0.0) set_objective_sense(model, MIN_SENSE) optimize!(model) - lower = objective_bound(model) + lower = min(objective_bound(model), 0.0) status = if upper <= 0 "stabily inactive" @@ -104,7 +112,7 @@ function calculate_bounds_fast(model::JuMP.Model, layer, neuron, W, b, neurons, else "normal" end - println("Neuron: $neuron, $status") + println("Neuron: $neuron, $status, bounds: [$lower, $upper]") return upper, lower end \ No newline at end of file diff --git a/src/neural_networks/compress.jl b/src/neural_networks/compress.jl index 4eb18a8..caea40c 100644 --- a/src/neural_networks/compress.jl +++ b/src/neural_networks/compress.jl @@ -2,181 +2,165 @@ using Flux using JuMP using LinearAlgebra: rank, dot -# function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}, params; big_M=1000.0) +function compress_fast(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}, params) -# K = length(model) - -# @assert all([model[i].σ == relu for i in 1:K-1]) "Neural network must use the relu activation function." -# @assert model[K].σ == identity "Neural network must use the identity function for the output layer." + K = length(model) -# W = deepcopy([Flux.params(model)[2*k-1] for k in 1:K]) # W[i] = weight matrix for i:th layer -# b = deepcopy([Flux.params(model)[2*k] for k in 1:K]) + @assert all([model[i].σ == relu for i in 1:K-1]) "Neural network must use the relu activation function." + @assert model[K].σ == identity "Neural network must use the identity function for the output layer." -# removed_neurons = Vector{Vector}(undef, K) -# [removed_neurons[layer] = Vector{Int}() for layer in 1:K] + W = deepcopy([Flux.params(model)[2*k-1] for k in 1:K]) # W[i] = weight matrix for i:th layer + b = deepcopy([Flux.params(model)[2*k] for k in 1:K]) -# input_length = Int((length(W[1]) / length(b[1]))) -# neuron_count = [length(b[k]) for k in eachindex(b)] -# neurons(layer) = layer == 0 ? [i for i in 1:input_length] : [i for i in setdiff(1:length(b[layer]), removed_neurons[layer])] + removed_neurons = Vector{Vector}(undef, K) + [removed_neurons[layer] = Vector{Int}() for layer in 1:K] -# @assert input_length == length(init_ub) == length(init_lb) "Initial bounds arrays must be the same length as the input layer" + input_length = Int((length(W[1]) / length(b[1]))) + neuron_count = [length(b[k]) for k in eachindex(b)] + neurons(layer) = layer == 0 ? [i for i in 1:input_length] : [i for i in setdiff(1:neuron_count[layer], removed_neurons[layer])] -# # build JuMP model -# jump_model = direct_model(Gurobi.Optimizer()) -# params.silent && set_silent(jump_model) -# params.threads != 0 && set_attribute(jump_model, "Threads", params.threads) -# params.relax && relax_integrality(jump_model) -# params.time_limit != 0 && set_attribute(jump_model, "TimeLimit", params.time_limit) + # build JuMP model + jump_model = direct_model(Gurobi.Optimizer()) + params.silent && set_silent(jump_model) + params.threads != 0 && set_attribute(jump_model, "Threads", params.threads) + params.relax && relax_integrality(jump_model) + params.time_limit != 0 && set_attribute(jump_model, "TimeLimit", params.time_limit) -# @variable(jump_model, x[layer = 0:K, neurons(layer)]) -# @variable(jump_model, s[layer = 1:K-1, neurons(layer)]) -# @variable(jump_model, z[layer = 1:K-1, neurons(layer)]) + @variable(jump_model, x[layer = 0:K, neurons(layer)]) + @variable(jump_model, s[layer = 1:K-1, neurons(layer)]) + @variable(jump_model, z[layer = 1:K-1, neurons(layer)]) -# @constraint(jump_model, [j = 1:input_length], x[0, j] <= init_ub[j]) -# @constraint(jump_model, [j = 1:input_length], x[0, j] >= init_lb[j]) + @constraint(jump_model, [j = 1:input_length], x[0, j] <= init_ub[j]) + @constraint(jump_model, [j = 1:input_length], x[0, j] >= init_lb[j]) -# bounds_U = Vector{Vector}(undef, K) -# bounds_L = Vector{Vector}(undef, K) + bounds_U = Vector{Vector}(undef, K) + bounds_L = Vector{Vector}(undef, K) -# layers_removed = 0 + layers_removed = 0 # how many strictly preceding layers have been removed at current loop iteration -# for layer in 1:K-1 # hidden layers + for layer in 1:K-1 # hidden layers -# println("\nLAYER: $layer") + println("\nLAYER $layer") -# bounds_U[layer] = fill(big_M, length(neurons(layer))) -# bounds_L[layer] = fill(big_M, length(neurons(layer))) + bounds_U[layer] = Vector{Float64}(undef, length(neurons(layer))) + bounds_L[layer] = Vector{Float64}(undef, length(neurons(layer))) -# bounds = map(neuron -> calculate_bounds_fast(jump_model, layer, neuron, W, b, neurons, layers_removed), neurons(layer)) -# bounds_U[layer], bounds_L[layer] = [bound[1] > big_M ? big_M : bound[1] for bound in bounds], [bound[2] > big_M ? big_M : bound[2] for bound in bounds] -# println() + bounds = map(neuron -> calculate_bounds_fast(jump_model, layer, neuron, W, b, neurons, layers_removed), neurons(layer)) + bounds_U[layer], bounds_L[layer] = [bound[1] for bound in bounds], [bound[2] for bound in bounds] -# stable_units = Set{Int}() # indices of stable neurons -# unstable_units = false + stable_units = Set{Int}() # indices of stable neurons + unstable_units = false -# for neuron in neurons(layer) + for neuron in 1:neuron_count[layer] -# if bounds_U[layer][neuron] <= 0 || iszero(W[layer][neuron, :]) # stabily inactive + if bounds_U[layer][neuron] <= 0 || iszero(W[layer][neuron, :]) # stabily inactive -# #println("Neuron $neuron is stabily inactive") - -# if neuron < last(neurons(layer)) || length(stable_units) > 0 || unstable_units == true + if neuron < neuron_count[layer] || length(stable_units) > 0 || unstable_units == true -# if iszero(W[layer][neuron, :]) && b[layer][neuron] > 0 - -# for neuron_next in neurons(layer+1) -# b[layer+1][neuron_next] += W[layer+1][neuron_next, neuron] * b[layer][neuron] -# end - -# end - -# push!(removed_neurons[layer], neuron) -# end + if iszero(W[layer][neuron, :]) && b[layer][neuron] > 0 + for neuron_next in 1:neuron_count[layer+1] # adjust biases + b[layer+1][neuron_next] += W[layer+1][neuron_next, neuron] * b[layer][neuron] + end + end -# elseif bounds_L[layer][neuron] >= 0 # stabily active + push!(removed_neurons[layer], neuron) + end -# #println("Neuron $neuron is stabily active") + elseif bounds_L[layer][neuron] >= 0 # stabily active -# if rank(W[layer][collect(union(stable_units, neuron)), :]) > length(stable_units) -# push!(stable_units, neuron) -# else -# S = sort!(collect(stable_units)) - -# alpha = transpose(W[layer][S, :]) \ W[layer][neuron, :] - -# @assert transpose(W[layer][S, :]) * alpha ≈ W[layer][neuron, :] "Alpha calculation not working." + if rank(W[layer][collect(union(stable_units, neuron)), :]) > length(stable_units) + push!(stable_units, neuron) + else # neuron is linearly dependent -# for neuron_next in neurons(layer+1) -# W[layer+1][neuron_next, S] .+= sum(W[layer+1][neuron_next, neuron] * alpha) -# b[layer+1][neuron_next] += W[layer+1][neuron_next, neuron] * (b[layer][neuron] + dot(b[layer][S], alpha)) -# end + S = collect(stable_units) + alpha = transpose(W[layer][S, :]) \ W[layer][neuron, :] -# push!(removed_neurons[layer], neuron) -# end -# else -# unstable_units = true -# end + for neuron_next in 1:neuron_count[layer+1] # adjust weights and biases + W[layer+1][neuron_next, S] .+= W[layer+1][neuron_next, neuron] * alpha + b[layer+1][neuron_next] += W[layer+1][neuron_next, neuron] * (b[layer][neuron] - dot(b[layer][S], alpha)) + end -# end + push!(removed_neurons[layer], neuron) + end + else + unstable_units = true + end -# println() + end -# if unstable_units == false # all units in the layer are stable -# println("Fully stable layer") + if unstable_units == false # all units in the layer are stable + println("Fully stable layer") -# if length(stable_units) > 0 + if length(stable_units) > 0 -# W_bar = Matrix{eltype(W[1][1])}(undef, neuron_count[layer+1], neuron_count[layer-1]) -# b_bar = Vector{eltype(b[1][1])}(undef, neuron_count[layer+1]) + W_bar = Matrix{eltype(W[1][1])}(undef, neuron_count[layer+1], neuron_count[layer-1-layers_removed]) + b_bar = Vector{eltype(b[1][1])}(undef, neuron_count[layer+1]) -# for neuron_next in neurons(layer+1) + S = collect(stable_units) -# b_bar[neuron_next] = b[layer+1][neuron_next] + sum([W[layer+1][neuron_next, k] * b[layer][k] for k in collect(stable_units)]) + for neuron_next in 1:neuron_count[layer+1] -# for neuron_previous in neurons(layer-1) -# W_bar[neuron_next, neuron_previous] = sum([W[layer+1][neuron_next, k] * W[layer][k, neuron_previous] for k in collect(stable_units)]) -# end -# end + b_bar[neuron_next] = b[layer+1][neuron_next] + dot(W[layer+1][neuron_next, S], b[layer][S]) -# W[layer+1] = W_bar -# b[layer+1] = b_bar + for neuron_previous in 1:neuron_count[layer-1-layers_removed] + W_bar[neuron_next, neuron_previous] = dot(W[layer+1][neuron_next, S], W[layer][S, neuron_previous]) + end + end -# layers_removed += 1 -# push!(removed_neurons[layer], neurons(layer)...) -# else -# output = model((init_ub + init_lb) ./ 2) -# println("WHOLE NETWORK IS CONSTANT WITH OUTPUT: $output") -# return -# end -# end + W[layer+1] = W_bar + b[layer+1] = b_bar -# println("Removed $(length(removed_neurons[layer]))/$(neuron_count[layer]) neurons") + layers_removed += 1 + push!(removed_neurons[layer], neurons(layer)...) + else + output = model((init_ub + init_lb) ./ 2) + println("WHOLE NETWORK IS CONSTANT WITH OUTPUT: $output") + return output + end + end -# for neuron in neurons(layer) -# @constraint(jump_model, x[layer, neuron] >= 0) -# @constraint(jump_model, s[layer, neuron] >= 0) -# set_binary(z[layer, neuron]) + println("Removed $(length(removed_neurons[layer]))/$(neuron_count[layer]) neurons") -# @constraint(jump_model, x[layer, neuron] <= bounds_U[layer][neuron] * (1 - z[layer, neuron])) -# @constraint(jump_model, s[layer, neuron] <= -bounds_L[layer][neuron] * z[layer, neuron]) + for neuron in neurons(layer) + @constraint(jump_model, x[layer, neuron] >= 0) + @constraint(jump_model, s[layer, neuron] >= 0) + set_binary(z[layer, neuron]) -# @constraint(jump_model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1-layers_removed, i] for i in neurons(layer-1-layers_removed))) -# end + @constraint(jump_model, x[layer, neuron] <= bounds_U[layer][neuron] * (1 - z[layer, neuron])) + @constraint(jump_model, s[layer, neuron] <= -bounds_L[layer][neuron] * z[layer, neuron]) -# if length(neurons(layer)) > 0 -# layers_removed = 0 -# end + @constraint(jump_model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1-layers_removed, i] for i in neurons(layer-1-layers_removed))) + end -# end + if length(neurons(layer)) > 0 + layers_removed = 0 + end -# # output layer -# @constraint(jump_model, [neuron in neurons(K)], x[K, neuron] == b[K][neuron] + sum(W[K][neuron, i] * x[K-1-layers_removed, i] for i in neurons(K-1-layers_removed))) + end -# println("Weights: $([size(W[layer]) for layer in 1:K])") + # output layer + @constraint(jump_model, [neuron in neurons(K)], x[K, neuron] == b[K][neuron] + sum(W[K][neuron, i] * x[K-1-layers_removed, i] for i in neurons(K-1-layers_removed))) -# # build compressed model -# new_layers = []; -# layers = findall(neurons -> length(neurons) > 0, [neurons(l) for l in 1:K]) # exclude layers with no neurons -# for (i, layer) in enumerate(layers) + # build compressed model + new_layers = []; + layers = findall(neurons -> length(neurons) > 0, [neurons(l) for l in 1:K]) # layers with neurons + for (i, layer) in enumerate(layers) -# if layers[i-1] != layer-1 # previous layer was removed -# else -# W[layer] = W[layer][setdiff(1:size(W[layer])[1], removed_neurons[layer]), setdiff(1:size(W[layer])[2], layer == 1 ? [] : removed_neurons[layers[i-1]])] -# b[layer] = b[layer][setdiff(1:length(b[layer]), removed_neurons[layer])] -# end + W[layer] = W[layer][neurons(layer), neurons(i == 1 ? 0 : layers[i-1])] + b[layer] = b[layer][neurons(layer)] -# if layer != last(layers) -# push!(new_layers, Dense(W[layer], b[layer], relu)) -# else -# push!(new_layers, Dense(W[layer], b[layer])) -# end -# end + if layer != last(layers) + push!(new_layers, Dense(W[layer], b[layer], relu)) + else + push!(new_layers, Dense(W[layer], b[layer])) + end + end -# new_model = Flux.Chain(new_layers...) + new_model = Flux.Chain(new_layers...) -# return jump_model, removed_neurons, new_model, bounds_U, bounds_L + return jump_model, removed_neurons, new_model, bounds_U, bounds_L -# end +end function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}, bounds_U, bounds_L) @@ -223,7 +207,7 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F if rank(W[layer][collect(union(stable_units, neuron)), :]) > length(stable_units) push!(stable_units, neuron) - else + else # neuron is linearly dependent S = collect(stable_units) alpha = transpose(W[layer][S, :]) \ W[layer][neuron, :] From 070dde9388f396fe656bb1450846416befd26b74 Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Fri, 9 Feb 2024 15:36:42 +0200 Subject: [PATCH 23/32] added GLPK to nn compression --- examples/nn_comp_test.jl | 55 ---------------------- examples/random_nn_comp_test.jl | 78 ++++++++++++++++++++++++++++++++ src/Gogeta.jl | 2 +- src/neural_networks/NN_to_MIP.jl | 36 +++++++++------ src/neural_networks/bounds.jl | 56 +++++++++++++---------- src/neural_networks/compress.jl | 26 +++++------ 6 files changed, 145 insertions(+), 108 deletions(-) delete mode 100644 examples/nn_comp_test.jl create mode 100644 examples/random_nn_comp_test.jl diff --git a/examples/nn_comp_test.jl b/examples/nn_comp_test.jl deleted file mode 100644 index f8d619b..0000000 --- a/examples/nn_comp_test.jl +++ /dev/null @@ -1,55 +0,0 @@ -using Flux -using Random -using Revise -using Gogeta -using Test -using Plots - -begin - Random.seed!(1234); - - model = Chain( - Dense(2 => 10, relu), - Dense(10 => 50, relu), - Dense(50 => 20, relu), - Dense(20 => 5, relu), - Dense(5 => 1) - ) -end - -init_U = [-0.5, 0.5]; -init_L = [-1.0, -1.0]; - -x1 = (rand(100) * (init_U[1] - init_L[1])) .+ init_L[1]; -x2 = (rand(100) * (init_U[2] - init_L[2])) .+ init_L[2]; -x = transpose(hcat(x1, x2)) .|> Float32; - -solver_params = SolverParams(silent=true, threads=0, relax=false, time_limit=0); - -# compress with compress fast -@time compression_results = compress_fast(model, init_U, init_L, solver_params); -jump_model, removed_neurons, compressed_model, bounds_U, bounds_L = compression_results; - -# test that jump model and compressed model produce same results as original -@test vec(model(x)) ≈ [forward_pass!(jump_model, x[:, i])[] for i in 1:size(x)[2]] -@test compressed_model(x) ≈ model(x) - -# perform bound tightening -@time nn_jump, U_correct, L_correct = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds=true); - -# test that created jump model is equal to the original -@test vec(model(x)) ≈ [forward_pass!(nn_jump, x[:, i])[] for i in 1:size(x)[2]] - -# compare bounds with/without fast compression -plot(collect(Iterators.flatten(bounds_U[1:end-1]))) -plot!(collect(Iterators.flatten(U_correct[1:end-1]))) - -plot!(collect(Iterators.flatten(bounds_L[1:end-1]))) -plot!(collect(Iterators.flatten(L_correct[1:end-1]))) - -# contour plot the model -x_range = LinRange(init_L[1], init_U[1], 100); -y_range = LinRange(init_L[2], init_U[2], 100); - -contourf(x_range, y_range, (x, y) -> model(hcat(x, y)')[], c=cgrad(:viridis), lw=0) -contourf(x_range, y_range, (x, y) -> compressed_model(hcat(x, y)')[], c=cgrad(:viridis), lw=0) \ No newline at end of file diff --git a/examples/random_nn_comp_test.jl b/examples/random_nn_comp_test.jl new file mode 100644 index 0000000..055de6a --- /dev/null +++ b/examples/random_nn_comp_test.jl @@ -0,0 +1,78 @@ +using Flux +using Random +using Revise +using Gogeta +using Test +using Plots + +begin + Random.seed!(1234); + + model = Chain( + Dense(2 => 10, relu), + Dense(10 => 50, relu), + Dense(50 => 20, relu), + Dense(20 => 5, relu), + Dense(5 => 1) + ) +end + +init_U = [-0.5, 0.5]; +init_L = [-1.5, -0.5]; + +x1 = (rand(100) * (init_U[1] - init_L[1])) .+ init_L[1]; +x2 = (rand(100) * (init_U[2] - init_L[2])) .+ init_L[2]; +x = transpose(hcat(x1, x2)) .|> Float32; + +solver_params = SolverParams(solver="GLPK", silent=true, threads=0, relax=false, time_limit=0); + +# compress with compress fast +@time compression_results = compress_and_tighten(model, init_U, init_L, solver_params); +jump_model, compressed_model, removed_neurons, bounds_U, bounds_L = compression_results; + +# test that jump model and compressed model produce same results as original +@test vec(model(x)) ≈ [forward_pass!(jump_model, x[:, i])[] for i in 1:size(x)[2]] +@test compressed_model(x) ≈ model(x) + +# perform bound tightening +@time nn_jump, U_correct, L_correct = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds=true); +# test that created jump model is equal to the original +@test vec(model(x)) ≈ [forward_pass!(nn_jump, x[:, i])[] for i in 1:size(x)[2]] + +# test compression with precomputed bounds +@time res = compress(model, init_U, init_L, U_correct, L_correct); +compressed, removed = res; + +@test compressed(x) ≈ model(x) + +removed +removed_neurons +compressed +compressed_model +@test compressed_model(x) ≈ compressed(x) + +# compare bounds with/without fast compression +U_data = collect(Iterators.flatten(bounds_U)); +U_true = collect(Iterators.flatten(U_correct)); + +L_data = collect(Iterators.flatten(bounds_L)); +L_true = collect(Iterators.flatten(L_correct)); + +plot(U_data) +plot!(U_true) + +plot!(L_data) +plot!(L_true) + +# test jump model without bound tightening +@time nn_jump, U_correct, L_correct = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds=false); +# test that created jump model is equal to the original +@test vec(model(x)) ≈ [forward_pass!(nn_jump, x[:, i])[] for i in 1:size(x)[2]] +plot(vec(model(x)) - [forward_pass!(nn_jump, x[:, i])[] for i in 1:size(x)[2]]) + +# contour plot the model +x_range = LinRange{Float32}(init_L[1], init_U[1], 100); +y_range = LinRange{Float32}(init_L[2], init_U[2], 100); + +contourf(x_range, y_range, (x, y) -> model(hcat(x, y)')[], c=cgrad(:viridis), lw=0) +contourf(x_range, y_range, (x, y) -> compressed_model(hcat(x, y)')[], c=cgrad(:viridis), lw=0) \ No newline at end of file diff --git a/src/Gogeta.jl b/src/Gogeta.jl index 5793825..fd0970e 100644 --- a/src/Gogeta.jl +++ b/src/Gogeta.jl @@ -6,7 +6,7 @@ export NN_to_MIP, forward_pass!, SolverParams include("neural_networks/bounds.jl") include("neural_networks/compress.jl") -export compress, compress_fast +export compress, compress_and_tighten include("tree_ensembles/types.jl") export TEModel, extract_evotrees_info diff --git a/src/neural_networks/NN_to_MIP.jl b/src/neural_networks/NN_to_MIP.jl index be01b12..6dfdd32 100644 --- a/src/neural_networks/NN_to_MIP.jl +++ b/src/neural_networks/NN_to_MIP.jl @@ -3,6 +3,7 @@ using JuMP using Distributed @kwdef struct SolverParams + solver::String silent::Bool threads::Int relax::Bool @@ -39,19 +40,19 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect bounds_L = Vector{Vector}(undef, K) for layer in 1:K # hidden layers and output - - bounds_U[layer] = Vector{Float64}(undef, length(neurons(layer))) - bounds_L[layer] = Vector{Float64}(undef, length(neurons(layer))) println("\nLAYER $layer") if tighten_bounds - bounds = if nprocs() > 1 - pmap(neuron -> calculate_bounds(copy_model(jump_model, solver_params), layer, neuron, W, b, neurons), neurons(layer)) - else - map(neuron -> calculate_bounds(jump_model, layer, neuron, W, b, neurons), neurons(layer)) - end + bounds = if nprocs() > 1 + pmap(neuron -> calculate_bounds(copy_model(jump_model, solver_params), layer, neuron, W, b, neurons), neurons(layer)) + else + map(neuron -> calculate_bounds(jump_model, layer, neuron, W, b, neurons), neurons(layer)) + end bounds_U[layer], bounds_L[layer] = [bound[1] for bound in bounds], [bound[2] for bound in bounds] + else + bounds_U[layer] = fill(BIG_M[], length(neurons(layer))) + bounds_L[layer] = fill(-BIG_M[], length(neurons(layer))) end if layer == K # output bounds calculated but no unnecessary constraints added @@ -79,12 +80,17 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect end function forward_pass!(jump_model::JuMP.Model, input) - @assert length(input) == length(jump_model[:x][0, :]) "Incorrect input length." - - [fix(jump_model[:x][0, i], input[i], force=true) for i in eachindex(input)] - optimize!(jump_model) + + try + @assert length(input) == length(jump_model[:x][0, :]) "Incorrect input length." + [fix(jump_model[:x][0, i], input[i], force=true) for i in eachindex(input)] + optimize!(jump_model) + (last_layer, outputs) = maximum(keys(jump_model[:x].data)) + result = value.(jump_model[:x][last_layer, :]) + return [result[i] for i in 1:outputs] + catch e + println("Input outside of input bounds or incorrectly constructed model.") + return [nothing] + end - (last_layer, outputs) = maximum(keys(jump_model[:x].data)) - result = value.(jump_model[:x][last_layer, :]) - return [result[i] for i in 1:outputs] end \ No newline at end of file diff --git a/src/neural_networks/bounds.jl b/src/neural_networks/bounds.jl index 22c7eed..44470bb 100644 --- a/src/neural_networks/bounds.jl +++ b/src/neural_networks/bounds.jl @@ -3,6 +3,7 @@ using Gurobi using GLPK const GUROBI_ENV = Ref{Gurobi.Env}() +global const BIG_M = Ref{Float64}(1.0e10) function __init__() const GUROBI_ENV[] = Gurobi.Env() @@ -15,23 +16,32 @@ function copy_model(input_model, solver_params) end function set_solver_params!(model, params) - set_optimizer(model, () -> Gurobi.Optimizer(GUROBI_ENV[])) - # set_optimizer(model, () -> GLPK.Optimizer()) + if params.solver == "Gurobi" + set_optimizer(model, () -> Gurobi.Optimizer(GUROBI_ENV[])) + params.time_limit != 0 && set_attribute(model, "TimeLimit", params.time_limit) + params.threads != 0 && set_attribute(model, "Threads", params.threads) + elseif params.solver == "GLPK" + set_optimizer(model, () -> GLPK.Optimizer()) + params.time_limit != 0 && set_attribute(model, "tm_lim", params.time_limit) + global BIG_M[] = 1_000.0 # GLPK cannot handle big-M constraints with large values + else + error("Solver has to be \"Gurobi\"/\"GLPK\"") + end + params.silent && set_silent(model) - params.threads != 0 && set_attribute(model, "Threads", params.threads) params.relax && relax_integrality(model) - params.time_limit != 0 && set_attribute(model, "TimeLimit", params.time_limit) - # params.time_limit != 0 && set_attribute(model, "tm_lim", params.time_limit) + end -function calculate_bounds(model::JuMP.Model, layer, neuron, W, b, neurons) +function calculate_bounds(model::JuMP.Model, layer, neuron, W, b, neurons; layers_removed=0) - @objective(model, Max, b[layer][neuron] + sum(W[layer][neuron, i] * model[:x][layer-1, i] for i in neurons(layer-1))) + @objective(model, Max, b[layer][neuron] + sum(W[layer][neuron, i] * model[:x][layer-1-layers_removed, i] for i in neurons(layer-1-layers_removed))) optimize!(model) upper_bound = if termination_status(model) == OPTIMAL max(objective_value(model), 0.0) else + @warn "Layer $layer, neuron $neuron could not be solved to optimality." max(objective_bound(model), 0.0) end @@ -41,6 +51,7 @@ function calculate_bounds(model::JuMP.Model, layer, neuron, W, b, neurons) lower_bound = if termination_status(model) == OPTIMAL min(objective_value(model), 0.0) else + @warn "Layer $layer, neuron $neuron could not be solved to optimality." min(objective_bound(model), 0.0) end @@ -51,41 +62,40 @@ end function calculate_bounds_fast(model::JuMP.Model, layer, neuron, W, b, neurons, layers_removed) + upper = 1.0e10 + lower = -1.0e10 + function bounds_callback(cb_data, cb_where::Cint) # Only run at integer solutions if cb_where == GRB_CB_MIPSOL objbound = Ref{Cdouble}() - objval = Ref{Cdouble}() + objbest = Ref{Cdouble}() GRBcbget(cb_data, cb_where, GRB_CB_MIPSOL_OBJBND, objbound) - GRBcbget(cb_data, cb_where, GRB_CB_MIPSOL_OBJ, objval) + GRBcbget(cb_data, cb_where, GRB_CB_MIPSOL_OBJBST, objbest) if objective_sense(model) == MAX_SENSE - println("UPPER: Bound: $(objbound[]), objective: $(objval[])") - - if objval[] > 0 - println("UPPER positive: $(objbound[])") + if objbest[] > 0 + upper = min(objbound[], 1.0e10) GRBterminate(backend(model)) end if objbound[] <= 0 - println("UPPER negative: $(objbound[])") + upper = max(objbound[], 0.0) GRBterminate(backend(model)) end elseif objective_sense(model) == MIN_SENSE - println("LOWER: Bound: $(objbound[]), objective: $(objval[])") - - if objval[] < 0 - println("LOWER negative: $(objbound[])") + if objbest[] < 0 + lower = max(objbound[], -1.0e10) GRBterminate(backend(model)) end if objbound[] >= 0 - println("LOWER positive: $(objbound[])") + lower = min(objbound[], 0.0) GRBterminate(backend(model)) end end @@ -95,15 +105,13 @@ function calculate_bounds_fast(model::JuMP.Model, layer, neuron, W, b, neurons, @objective(model, Max, b[layer][neuron] + sum(W[layer][neuron, i] * model[:x][layer-1-layers_removed, i] for i in neurons(layer-1-layers_removed))) - #set_attribute(model, "LazyConstraints", 1) - #set_attribute(model, Gurobi.CallbackFunction(), bounds_callback) + set_attribute(model, "LazyConstraints", 1) + set_attribute(model, Gurobi.CallbackFunction(), bounds_callback) optimize!(model) - upper = max(objective_bound(model), 0.0) set_objective_sense(model, MIN_SENSE) optimize!(model) - lower = min(objective_bound(model), 0.0) status = if upper <= 0 "stabily inactive" @@ -114,5 +122,7 @@ function calculate_bounds_fast(model::JuMP.Model, layer, neuron, W, b, neurons, end println("Neuron: $neuron, $status, bounds: [$lower, $upper]") + set_attribute(jump_model, Gurobi.CallbackFunction(), (cb_data, cb_where::Cint)->nothing) + return upper, lower end \ No newline at end of file diff --git a/src/neural_networks/compress.jl b/src/neural_networks/compress.jl index caea40c..d48aca6 100644 --- a/src/neural_networks/compress.jl +++ b/src/neural_networks/compress.jl @@ -2,7 +2,7 @@ using Flux using JuMP using LinearAlgebra: rank, dot -function compress_fast(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}, params) +function compress_and_tighten(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}, params) K = length(model) @@ -20,11 +20,8 @@ function compress_fast(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vec neurons(layer) = layer == 0 ? [i for i in 1:input_length] : [i for i in setdiff(1:neuron_count[layer], removed_neurons[layer])] # build JuMP model - jump_model = direct_model(Gurobi.Optimizer()) - params.silent && set_silent(jump_model) - params.threads != 0 && set_attribute(jump_model, "Threads", params.threads) - params.relax && relax_integrality(jump_model) - params.time_limit != 0 && set_attribute(jump_model, "TimeLimit", params.time_limit) + jump_model = Model() + set_solver_params!(jump_model, params) @variable(jump_model, x[layer = 0:K, neurons(layer)]) @variable(jump_model, s[layer = 1:K-1, neurons(layer)]) @@ -38,16 +35,17 @@ function compress_fast(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vec layers_removed = 0 # how many strictly preceding layers have been removed at current loop iteration - for layer in 1:K-1 # hidden layers + for layer in 1:K # hidden layers and bounds for output layer println("\nLAYER $layer") - bounds_U[layer] = Vector{Float64}(undef, length(neurons(layer))) - bounds_L[layer] = Vector{Float64}(undef, length(neurons(layer))) - - bounds = map(neuron -> calculate_bounds_fast(jump_model, layer, neuron, W, b, neurons, layers_removed), neurons(layer)) + bounds = map(neuron -> calculate_bounds(jump_model, layer, neuron, W, b, neurons; layers_removed), neurons(layer)) bounds_U[layer], bounds_L[layer] = [bound[1] for bound in bounds], [bound[2] for bound in bounds] + if layer == K + break + end + stable_units = Set{Int}() # indices of stable neurons unstable_units = false @@ -157,8 +155,8 @@ function compress_fast(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vec end new_model = Flux.Chain(new_layers...) - - return jump_model, removed_neurons, new_model, bounds_U, bounds_L + + return jump_model, new_model, removed_neurons, bounds_U, bounds_L end @@ -281,6 +279,6 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F new_model = Flux.Chain(new_layers...) - return nothing, removed_neurons, new_model, bounds_U, bounds_L + return new_model, removed_neurons end \ No newline at end of file From 159378042be2a2ea9b677e6d5b1489eabd6ddbe4 Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Fri, 9 Feb 2024 16:02:39 +0200 Subject: [PATCH 24/32] refactoring 'compress' and package contents --- examples/random_nn_comp_test.jl | 17 +- src/Gogeta.jl | 12 +- src/neural_networks/NN_to_MIP.jl | 4 - src/neural_networks/bounds.jl | 5 - src/neural_networks/compress.jl | 189 ++++-------------- .../{ => old}/lossless_compression.jl | 0 .../{ => old}/pretrained_model_pruner.jl | 0 src/tree_ensembles/TE_to_MIP.jl | 2 - src/tree_ensembles/types.jl | 2 - 9 files changed, 58 insertions(+), 173 deletions(-) rename src/neural_networks/{ => old}/lossless_compression.jl (100%) rename src/neural_networks/{ => old}/pretrained_model_pruner.jl (100%) diff --git a/examples/random_nn_comp_test.jl b/examples/random_nn_comp_test.jl index 055de6a..7c5ac31 100644 --- a/examples/random_nn_comp_test.jl +++ b/examples/random_nn_comp_test.jl @@ -26,8 +26,8 @@ x = transpose(hcat(x1, x2)) .|> Float32; solver_params = SolverParams(solver="GLPK", silent=true, threads=0, relax=false, time_limit=0); -# compress with compress fast -@time compression_results = compress_and_tighten(model, init_U, init_L, solver_params); +# compress with simultaneous bound tightening +@time compression_results = compress(model, init_U, init_L; params=solver_params); jump_model, compressed_model, removed_neurons, bounds_U, bounds_L = compression_results; # test that jump model and compressed model produce same results as original @@ -39,17 +39,16 @@ jump_model, compressed_model, removed_neurons, bounds_U, bounds_L = compression_ # test that created jump model is equal to the original @test vec(model(x)) ≈ [forward_pass!(nn_jump, x[:, i])[] for i in 1:size(x)[2]] -# test compression with precomputed bounds -@time res = compress(model, init_U, init_L, U_correct, L_correct); +# compress with precomputed bounds +@time res = compress(model, init_U, init_L; bounds_U=U_correct, bounds_L=L_correct); compressed, removed = res; @test compressed(x) ≈ model(x) +@test removed ≈ removed_neurons -removed -removed_neurons -compressed -compressed_model -@test compressed_model(x) ≈ compressed(x) +""" +VISUALIZATION +""" # compare bounds with/without fast compression U_data = collect(Iterators.flatten(bounds_U)); diff --git a/src/Gogeta.jl b/src/Gogeta.jl index fd0970e..3d485a7 100644 --- a/src/Gogeta.jl +++ b/src/Gogeta.jl @@ -1,12 +1,22 @@ module Gogeta +using Flux +using JuMP +using LinearAlgebra: rank, dot +using Gurobi +using GLPK +using Distributed +using EvoTrees + +const GUROBI_ENV = Ref{Gurobi.Env}() + include("neural_networks/NN_to_MIP.jl") export NN_to_MIP, forward_pass!, SolverParams include("neural_networks/bounds.jl") include("neural_networks/compress.jl") -export compress, compress_and_tighten +export compress include("tree_ensembles/types.jl") export TEModel, extract_evotrees_info diff --git a/src/neural_networks/NN_to_MIP.jl b/src/neural_networks/NN_to_MIP.jl index 6dfdd32..e1f14a6 100644 --- a/src/neural_networks/NN_to_MIP.jl +++ b/src/neural_networks/NN_to_MIP.jl @@ -1,7 +1,3 @@ -using Flux -using JuMP -using Distributed - @kwdef struct SolverParams solver::String silent::Bool diff --git a/src/neural_networks/bounds.jl b/src/neural_networks/bounds.jl index 44470bb..b592ced 100644 --- a/src/neural_networks/bounds.jl +++ b/src/neural_networks/bounds.jl @@ -1,8 +1,3 @@ -using JuMP -using Gurobi -using GLPK - -const GUROBI_ENV = Ref{Gurobi.Env}() global const BIG_M = Ref{Float64}(1.0e10) function __init__() diff --git a/src/neural_networks/compress.jl b/src/neural_networks/compress.jl index d48aca6..a0091bd 100644 --- a/src/neural_networks/compress.jl +++ b/src/neural_networks/compress.jl @@ -1,8 +1,7 @@ -using Flux -using JuMP -using LinearAlgebra: rank, dot +function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}; params=nothing, bounds_U=nothing, bounds_L=nothing) -function compress_and_tighten(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}, params) + with_tightening = (bounds_U === nothing || bounds_L === nothing) ? true : false + with_tightening && @assert params !== nothing "Solver parameters must be provided." K = length(model) @@ -20,18 +19,20 @@ function compress_and_tighten(model::Flux.Chain, init_ub::Vector{Float64}, init_ neurons(layer) = layer == 0 ? [i for i in 1:input_length] : [i for i in setdiff(1:neuron_count[layer], removed_neurons[layer])] # build JuMP model - jump_model = Model() - set_solver_params!(jump_model, params) - - @variable(jump_model, x[layer = 0:K, neurons(layer)]) - @variable(jump_model, s[layer = 1:K-1, neurons(layer)]) - @variable(jump_model, z[layer = 1:K-1, neurons(layer)]) - - @constraint(jump_model, [j = 1:input_length], x[0, j] <= init_ub[j]) - @constraint(jump_model, [j = 1:input_length], x[0, j] >= init_lb[j]) - - bounds_U = Vector{Vector}(undef, K) - bounds_L = Vector{Vector}(undef, K) + if with_tightening + jump_model = Model() + set_solver_params!(jump_model, params) + + @variable(jump_model, x[layer = 0:K, neurons(layer)]) + @variable(jump_model, s[layer = 1:K-1, neurons(layer)]) + @variable(jump_model, z[layer = 1:K-1, neurons(layer)]) + + @constraint(jump_model, [j = 1:input_length], x[0, j] <= init_ub[j]) + @constraint(jump_model, [j = 1:input_length], x[0, j] >= init_lb[j]) + + bounds_U = Vector{Vector}(undef, K) + bounds_L = Vector{Vector}(undef, K) + end layers_removed = 0 # how many strictly preceding layers have been removed at current loop iteration @@ -39,8 +40,14 @@ function compress_and_tighten(model::Flux.Chain, init_ub::Vector{Float64}, init_ println("\nLAYER $layer") - bounds = map(neuron -> calculate_bounds(jump_model, layer, neuron, W, b, neurons; layers_removed), neurons(layer)) - bounds_U[layer], bounds_L[layer] = [bound[1] for bound in bounds], [bound[2] for bound in bounds] + if with_tightening + bounds = if nprocs() > 1 + pmap(neuron -> calculate_bounds(copy_model(jump_model, solver_params), layer, neuron, W, b, neurons; layers_removed), neurons(layer)) + else + map(neuron -> calculate_bounds(jump_model, layer, neuron, W, b, neurons; layers_removed), neurons(layer)) + end + bounds_U[layer], bounds_L[layer] = [bound[1] for bound in bounds], [bound[2] for bound in bounds] + end if layer == K break @@ -119,15 +126,17 @@ function compress_and_tighten(model::Flux.Chain, init_ub::Vector{Float64}, init_ println("Removed $(length(removed_neurons[layer]))/$(neuron_count[layer]) neurons") - for neuron in neurons(layer) - @constraint(jump_model, x[layer, neuron] >= 0) - @constraint(jump_model, s[layer, neuron] >= 0) - set_binary(z[layer, neuron]) + if with_tightening + for neuron in neurons(layer) + @constraint(jump_model, x[layer, neuron] >= 0) + @constraint(jump_model, s[layer, neuron] >= 0) + set_binary(z[layer, neuron]) - @constraint(jump_model, x[layer, neuron] <= bounds_U[layer][neuron] * (1 - z[layer, neuron])) - @constraint(jump_model, s[layer, neuron] <= -bounds_L[layer][neuron] * z[layer, neuron]) + @constraint(jump_model, x[layer, neuron] <= bounds_U[layer][neuron] * (1 - z[layer, neuron])) + @constraint(jump_model, s[layer, neuron] <= -bounds_L[layer][neuron] * z[layer, neuron]) - @constraint(jump_model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1-layers_removed, i] for i in neurons(layer-1-layers_removed))) + @constraint(jump_model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1-layers_removed, i] for i in neurons(layer-1-layers_removed))) + end end if length(neurons(layer)) > 0 @@ -137,7 +146,7 @@ function compress_and_tighten(model::Flux.Chain, init_ub::Vector{Float64}, init_ end # output layer - @constraint(jump_model, [neuron in neurons(K)], x[K, neuron] == b[K][neuron] + sum(W[K][neuron, i] * x[K-1-layers_removed, i] for i in neurons(K-1-layers_removed))) + with_tightening && @constraint(jump_model, [neuron in neurons(K)], x[K, neuron] == b[K][neuron] + sum(W[K][neuron, i] * x[K-1-layers_removed, i] for i in neurons(K-1-layers_removed))) # build compressed model new_layers = []; @@ -155,130 +164,10 @@ function compress_and_tighten(model::Flux.Chain, init_ub::Vector{Float64}, init_ end new_model = Flux.Chain(new_layers...) - - return jump_model, new_model, removed_neurons, bounds_U, bounds_L - -end - -function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}, bounds_U, bounds_L) - - K = length(model) - - @assert all([model[i].σ == relu for i in 1:K-1]) "Neural network must use the relu activation function." - @assert model[K].σ == identity "Neural network must use the identity function for the output layer." - - W = deepcopy([Flux.params(model)[2*k-1] for k in 1:K]) # W[i] = weight matrix for i:th layer - b = deepcopy([Flux.params(model)[2*k] for k in 1:K]) - - removed_neurons = Vector{Vector}(undef, K) - [removed_neurons[layer] = Vector{Int}() for layer in 1:K] - - input_length = Int((length(W[1]) / length(b[1]))) - neuron_count = [length(b[k]) for k in eachindex(b)] - neurons(layer) = layer == 0 ? [i for i in 1:input_length] : [i for i in setdiff(1:neuron_count[layer], removed_neurons[layer])] - - layers_removed = 0 # how many strictly preceding layers have been removed at current loop iteration - - for layer in 1:K-1 # hidden layers - - println("\nLAYER $layer") - - stable_units = Set{Int}() # indices of stable neurons - unstable_units = false - - for neuron in 1:neuron_count[layer] - - if bounds_U[layer][neuron] <= 0 || iszero(W[layer][neuron, :]) # stabily inactive - - if neuron < neuron_count[layer] || length(stable_units) > 0 || unstable_units == true - - if iszero(W[layer][neuron, :]) && b[layer][neuron] > 0 - for neuron_next in 1:neuron_count[layer+1] # adjust biases - b[layer+1][neuron_next] += W[layer+1][neuron_next, neuron] * b[layer][neuron] - end - end - - push!(removed_neurons[layer], neuron) - end - - elseif bounds_L[layer][neuron] >= 0 # stabily active - - if rank(W[layer][collect(union(stable_units, neuron)), :]) > length(stable_units) - push!(stable_units, neuron) - else # neuron is linearly dependent - - S = collect(stable_units) - alpha = transpose(W[layer][S, :]) \ W[layer][neuron, :] - - for neuron_next in 1:neuron_count[layer+1] # adjust weights and biases - W[layer+1][neuron_next, S] .+= W[layer+1][neuron_next, neuron] * alpha - b[layer+1][neuron_next] += W[layer+1][neuron_next, neuron] * (b[layer][neuron] - dot(b[layer][S], alpha)) - end - - push!(removed_neurons[layer], neuron) - end - else - unstable_units = true - end - - end - - if unstable_units == false # all units in the layer are stable - println("Fully stable layer") - - if length(stable_units) > 0 - - W_bar = Matrix{eltype(W[1][1])}(undef, neuron_count[layer+1], neuron_count[layer-1-layers_removed]) - b_bar = Vector{eltype(b[1][1])}(undef, neuron_count[layer+1]) - - S = collect(stable_units) - - for neuron_next in 1:neuron_count[layer+1] - - b_bar[neuron_next] = b[layer+1][neuron_next] + dot(W[layer+1][neuron_next, S], b[layer][S]) - - for neuron_previous in 1:neuron_count[layer-1-layers_removed] - W_bar[neuron_next, neuron_previous] = dot(W[layer+1][neuron_next, S], W[layer][S, neuron_previous]) - end - end - - W[layer+1] = W_bar - b[layer+1] = b_bar - - layers_removed += 1 - push!(removed_neurons[layer], neurons(layer)...) - else - output = model((init_ub + init_lb) ./ 2) - println("WHOLE NETWORK IS CONSTANT WITH OUTPUT: $output") - return output - end - end - - println("Removed $(length(removed_neurons[layer]))/$(neuron_count[layer]) neurons") - - if length(neurons(layer)) > 0 - layers_removed = 0 - end + if with_tightening + return jump_model, new_model, removed_neurons, bounds_U, bounds_L + else + return new_model, removed_neurons end - - # build compressed model - new_layers = []; - layers = findall(neurons -> length(neurons) > 0, [neurons(l) for l in 1:K]) # layers with neurons - for (i, layer) in enumerate(layers) - - W[layer] = W[layer][neurons(layer), neurons(i == 1 ? 0 : layers[i-1])] - b[layer] = b[layer][neurons(layer)] - - if layer != last(layers) - push!(new_layers, Dense(W[layer], b[layer], relu)) - else - push!(new_layers, Dense(W[layer], b[layer])) - end - end - - new_model = Flux.Chain(new_layers...) - - return new_model, removed_neurons - end \ No newline at end of file diff --git a/src/neural_networks/lossless_compression.jl b/src/neural_networks/old/lossless_compression.jl similarity index 100% rename from src/neural_networks/lossless_compression.jl rename to src/neural_networks/old/lossless_compression.jl diff --git a/src/neural_networks/pretrained_model_pruner.jl b/src/neural_networks/old/pretrained_model_pruner.jl similarity index 100% rename from src/neural_networks/pretrained_model_pruner.jl rename to src/neural_networks/old/pretrained_model_pruner.jl diff --git a/src/tree_ensembles/TE_to_MIP.jl b/src/tree_ensembles/TE_to_MIP.jl index a4408d8..6e7c00c 100755 --- a/src/tree_ensembles/TE_to_MIP.jl +++ b/src/tree_ensembles/TE_to_MIP.jl @@ -1,5 +1,3 @@ -using JuMP - """ ```julia function TE_to_MIP(TE::TEModel, optimizer, objective) diff --git a/src/tree_ensembles/types.jl b/src/tree_ensembles/types.jl index 19160e7..87f484f 100755 --- a/src/tree_ensembles/types.jl +++ b/src/tree_ensembles/types.jl @@ -1,5 +1,3 @@ -using EvoTrees - """ Universal datatype for storing information about a Tree Ensemble Model. This is the datatype that is used when creating the integer optimization problem from a tree ensemble. From 38778a15d430ae7b678b3fa63ba8057d3731bcb3 Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Tue, 13 Feb 2024 17:38:27 +0200 Subject: [PATCH 25/32] added bound heuristic tightening LRR --- examples/random_nn_comp_test.jl | 17 +++++--- examples/vime_nn_test.jl | 70 ++++++++++++++++++++++++++++++++ src/Gogeta.jl | 4 ++ src/neural_networks/NN_to_MIP.jl | 13 ++++-- src/neural_networks/bounds.jl | 20 ++++----- src/neural_networks/compress.jl | 4 +- 6 files changed, 105 insertions(+), 23 deletions(-) create mode 100644 examples/vime_nn_test.jl diff --git a/examples/random_nn_comp_test.jl b/examples/random_nn_comp_test.jl index 7c5ac31..42558be 100644 --- a/examples/random_nn_comp_test.jl +++ b/examples/random_nn_comp_test.jl @@ -46,6 +46,12 @@ compressed, removed = res; @test compressed(x) ≈ model(x) @test removed ≈ removed_neurons +# test jump model without bound tightening +@time nn_loose, U_loose, L_loose = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds=false); + +# test that created jump model is equal to the original +@test vec(model(x)) ≈ [forward_pass!(nn_loose, x[:, i])[] for i in 1:size(x)[2]] + """ VISUALIZATION """ @@ -63,11 +69,12 @@ plot!(U_true) plot!(L_data) plot!(L_true) -# test jump model without bound tightening -@time nn_jump, U_correct, L_correct = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds=false); -# test that created jump model is equal to the original -@test vec(model(x)) ≈ [forward_pass!(nn_jump, x[:, i])[] for i in 1:size(x)[2]] -plot(vec(model(x)) - [forward_pass!(nn_jump, x[:, i])[] for i in 1:size(x)[2]]) +# compare bounds with/without loose tightening +plot(collect(Iterators.flatten(U_correct))) +plot!(collect(Iterators.flatten(U_loose))) + +plot!(collect(Iterators.flatten(L_correct))) +plot!(collect(Iterators.flatten(L_loose))) # contour plot the model x_range = LinRange{Float32}(init_L[1], init_U[1], 100); diff --git a/examples/vime_nn_test.jl b/examples/vime_nn_test.jl new file mode 100644 index 0000000..47095ef --- /dev/null +++ b/examples/vime_nn_test.jl @@ -0,0 +1,70 @@ +using Flux +using Random +using Revise +using Gogeta +using Test +using Plots +using NPZ + +n_neurons = Int64[2, 1024, 512, 512, 256, 1] +n_neurons_cumulative_indices = [i+1 for i in [0, cumsum(n_neurons)...]] +model = load_model(n_neurons, "/Users/eetureijonen/Desktop/GAMMA-OPT/Gogeta.jl/src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0") + +init_U = [-0.5, 0.5]; +init_L = [-1.5, -0.5]; + +x1 = (rand(10) * (init_U[1] - init_L[1])) .+ init_L[1]; +x2 = (rand(10) * (init_U[2] - init_L[2])) .+ init_L[2]; +x = transpose(hcat(x1, x2)) .|> Float32; + +solver_params = SolverParams(solver="Gurobi", silent=true, threads=0, relax=false, time_limit=0); + +# test jump model using only fast bound tightening +@time nn_loose, U_loose, L_loose = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds=false); + +# test that created jump model is equal to the original (0.1% tolerance) +@test isapprox(vec(model(x)), [forward_pass!(nn_loose, x[:, i])[] for i in 1:size(x)[2]]; rtol=0.001) + +plot([forward_pass!(nn_loose, x[:, i])[] for i in 1:size(x)[2]] - vec(model(x))) +plot!(vec(model(x))) + +plot(relu.(collect(Iterators.flatten(U_loose))[1:end-1]) .+ 0.01)#, yscale=:log10) + +U_data, L_data = get_bounds("/Users/eetureijonen/Desktop/GAMMA-OPT/Gogeta.jl/src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0"); +U_data = U_data[3:end]; +L_data = L_data[3:end]; + +plot!(relu.(U_data[1:end-1]) .+ 0.01) + + +# compress with precomputed loose bounds +@time compressed_loose, removed_loose = compress(model, init_U, init_L; bounds_U=U_loose, bounds_L=L_loose); + +# test that compressed model is same as the original +@test compressed_loose(x) ≈ model(x) + +b = [Flux.params(model)[2*k] for k in 1:length(model)]; +neuron_count = [length(b[k]) for k in eachindex(b)]; + +U_full = Vector{Vector}(undef, length(model)); +L_full = Vector{Vector}(undef, length(model)); + +[U_full[layer] = Vector{Float64}(undef, neuron_count[layer]) for layer in 1:length(model)]; +[L_full[layer] = Vector{Float64}(undef, neuron_count[layer]) for layer in 1:length(model)]; + +for layer in 1:length(model) + for neuron in 1:neuron_count[layer] + U_full[layer][neuron] = U_data[neuron + (layer == 1 ? 0 : cumsum(neuron_count)[layer-1])] + L_full[layer][neuron] = L_data[neuron + (layer == 1 ? 0 : cumsum(neuron_count)[layer-1])] + end +end + +collect(Iterators.flatten(U_full)) == U_data +collect(Iterators.flatten(L_full)) == L_data + +# compress with precomputed LP bounds +@time res = compress(model, init_U, init_L; bounds_U=U_full, bounds_L=L_full); +compressed, removed = res; + +# test that compressed model is same as the original +@test compressed(x) ≈ model(x) diff --git a/src/Gogeta.jl b/src/Gogeta.jl index 3d485a7..551b0bb 100644 --- a/src/Gogeta.jl +++ b/src/Gogeta.jl @@ -10,6 +10,10 @@ using EvoTrees const GUROBI_ENV = Ref{Gurobi.Env}() +function __init__() + const GUROBI_ENV[] = Gurobi.Env() +end + include("neural_networks/NN_to_MIP.jl") export NN_to_MIP, forward_pass!, SolverParams diff --git a/src/neural_networks/NN_to_MIP.jl b/src/neural_networks/NN_to_MIP.jl index e1f14a6..4664853 100644 --- a/src/neural_networks/NN_to_MIP.jl +++ b/src/neural_networks/NN_to_MIP.jl @@ -47,8 +47,13 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect end bounds_U[layer], bounds_L[layer] = [bound[1] for bound in bounds], [bound[2] for bound in bounds] else - bounds_U[layer] = fill(BIG_M[], length(neurons(layer))) - bounds_L[layer] = fill(-BIG_M[], length(neurons(layer))) + if layer == 1 + bounds_U[layer] = [sum(max(W[layer][neuron, previous] * init_ub[previous], W[layer][neuron, previous] * init_lb[previous]) for previous in neurons(layer-1)) + b[layer][neuron] for neuron in neurons(layer)] + bounds_L[layer] = [sum(min(W[layer][neuron, previous] * init_ub[previous], W[layer][neuron, previous] * init_lb[previous]) for previous in neurons(layer-1)) + b[layer][neuron] for neuron in neurons(layer)] + else + bounds_U[layer] = [sum(max(W[layer][neuron, previous] * max(0, bounds_U[layer-1][previous]), W[layer][neuron, previous] * max(0, bounds_L[layer-1][previous])) for previous in neurons(layer-1)) + b[layer][neuron] for neuron in neurons(layer)] + bounds_L[layer] = [sum(min(W[layer][neuron, previous] * max(0, bounds_U[layer-1][previous]), W[layer][neuron, previous] * max(0, bounds_L[layer-1][previous])) for previous in neurons(layer-1)) + b[layer][neuron] for neuron in neurons(layer)] + end end if layer == K # output bounds calculated but no unnecessary constraints added @@ -61,8 +66,8 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect @constraint(jump_model, s[layer, neuron] >= 0) set_binary(z[layer, neuron]) - @constraint(jump_model, x[layer, neuron] <= bounds_U[layer][neuron] * (1 - z[layer, neuron])) - @constraint(jump_model, s[layer, neuron] <= -bounds_L[layer][neuron] * z[layer, neuron]) + @constraint(jump_model, x[layer, neuron] <= max(0, bounds_U[layer][neuron]) * (1 - z[layer, neuron])) + @constraint(jump_model, s[layer, neuron] <= max(0, -bounds_L[layer][neuron]) * z[layer, neuron]) @constraint(jump_model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1, i] for i in neurons(layer-1))) diff --git a/src/neural_networks/bounds.jl b/src/neural_networks/bounds.jl index b592ced..4ee0ea3 100644 --- a/src/neural_networks/bounds.jl +++ b/src/neural_networks/bounds.jl @@ -1,9 +1,3 @@ -global const BIG_M = Ref{Float64}(1.0e10) - -function __init__() - const GUROBI_ENV[] = Gurobi.Env() -end - function copy_model(input_model, solver_params) model = copy(input_model) set_solver_params!(model, solver_params) @@ -18,9 +12,8 @@ function set_solver_params!(model, params) elseif params.solver == "GLPK" set_optimizer(model, () -> GLPK.Optimizer()) params.time_limit != 0 && set_attribute(model, "tm_lim", params.time_limit) - global BIG_M[] = 1_000.0 # GLPK cannot handle big-M constraints with large values else - error("Solver has to be \"Gurobi\"/\"GLPK\"") + error("Solver has to be \"Gurobi\" or \"GLPK\"") end params.silent && set_silent(model) @@ -34,22 +27,25 @@ function calculate_bounds(model::JuMP.Model, layer, neuron, W, b, neurons; layer optimize!(model) upper_bound = if termination_status(model) == OPTIMAL - max(objective_value(model), 0.0) + objective_value(model) else @warn "Layer $layer, neuron $neuron could not be solved to optimality." - max(objective_bound(model), 0.0) + objective_bound(model) end set_objective_sense(model, MIN_SENSE) optimize!(model) lower_bound = if termination_status(model) == OPTIMAL - min(objective_value(model), 0.0) + objective_value(model) else @warn "Layer $layer, neuron $neuron could not be solved to optimality." - min(objective_bound(model), 0.0) + objective_bound(model) end + if upper_bound > 1_000 @warn "Upper bound is very loose: $upper_bound, problem might become infeasible." end + if lower_bound < -1_000 @warn "Lower bound is very loose: $lower_bound, problem might become infeasible." end + println("Neuron: $neuron") return upper_bound, lower_bound diff --git a/src/neural_networks/compress.jl b/src/neural_networks/compress.jl index a0091bd..5d0fdf5 100644 --- a/src/neural_networks/compress.jl +++ b/src/neural_networks/compress.jl @@ -132,8 +132,8 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F @constraint(jump_model, s[layer, neuron] >= 0) set_binary(z[layer, neuron]) - @constraint(jump_model, x[layer, neuron] <= bounds_U[layer][neuron] * (1 - z[layer, neuron])) - @constraint(jump_model, s[layer, neuron] <= -bounds_L[layer][neuron] * z[layer, neuron]) + @constraint(jump_model, x[layer, neuron] <= max(0, bounds_U[layer][neuron]) * (1 - z[layer, neuron])) + @constraint(jump_model, s[layer, neuron] <= max(0, -bounds_L[layer][neuron]) * z[layer, neuron]) @constraint(jump_model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1-layers_removed, i] for i in neurons(layer-1-layers_removed))) end From 4bbd95f927ca39d88b1b8c49b407da20858ee15b Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Wed, 14 Feb 2024 17:43:55 +0200 Subject: [PATCH 26/32] tests with a large nn --- examples/images/lp_vs_lrr_bounds.svg | 43 +++++++++++++ examples/images/lp_vs_lrr_bounds_log.svg | 43 +++++++++++++ examples/large_nn_results.md | 77 ++++++++++++++++++++++++ examples/random_nn_comp_test.jl | 14 ++--- examples/vime_nn_test.jl | 16 +++-- 5 files changed, 179 insertions(+), 14 deletions(-) create mode 100644 examples/images/lp_vs_lrr_bounds.svg create mode 100644 examples/images/lp_vs_lrr_bounds_log.svg create mode 100644 examples/large_nn_results.md diff --git a/examples/images/lp_vs_lrr_bounds.svg b/examples/images/lp_vs_lrr_bounds.svg new file mode 100644 index 0000000..11fba09 --- /dev/null +++ b/examples/images/lp_vs_lrr_bounds.svg @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/examples/images/lp_vs_lrr_bounds_log.svg b/examples/images/lp_vs_lrr_bounds_log.svg new file mode 100644 index 0000000..f40c3ff --- /dev/null +++ b/examples/images/lp_vs_lrr_bounds_log.svg @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/examples/large_nn_results.md b/examples/large_nn_results.md new file mode 100644 index 0000000..4909742 --- /dev/null +++ b/examples/large_nn_results.md @@ -0,0 +1,77 @@ +# Compression and bound tightening performance in a large neural network + +**Model:** *Adadelta* $l_1=0.001$ $l_2=0.001$ *(from Vilhelm's thesis)* + +**Neurons:** *2 - 1024 - 512 - 512 - 256 - 1* + +## Computational performance + +### LRR bound tightening *(Grimstad and Andresson)* +Only considers the previous layer bounds and the weights. + +`0.626957 seconds (23.18 M allocations: 634.177 MiB, 11.46% gc time)` + +### Compression with LRR bounds +`0.219787 seconds (3.68 M allocations: 156.574 MiB, 7.72% gc time)` + +### LP bound tightening +Linear relaxation of the standard bound tightening. + +`(25 minutes on Vilhelm's computer)` + +### Compression with LP bounds +`0.474099 seconds (3.68 M allocations: 312.358 MiB, 5.48% gc time)` + +It cannot be seen from this data but the input domain can significantly affect the performance of the bound tightening. + +### Comparison of the upper bounds +LP bounds shown in red and LRR bounds in blue. + + + + +The bounds get progressively worse with the deeper layers. First layer bounds are very similar but there is about a five-fold difference for the last layers. + +## Model compression + +`model` is the original. `compressed_loose` corresponds to the model compressed with LRR bounds. `compressed` corresponds to the the model compressed with LP bounds. + +```julia +julia> model +Chain( + Dense(2 => 1024, relu), # 3_072 parameters + Dense(1024 => 512, relu), # 524_800 parameters + Dense(512 => 512, relu), # 262_656 parameters + Dense(512 => 256, relu), # 131_328 parameters + Dense(256 => 1), # 257 parameters +) # Total: 10 arrays, 922_113 parameters, 7.036 MiB. + +julia> compressed_loose +Chain( + Dense(2 => 132, relu), # 396 parameters + Dense(132 => 439, relu), # 58_387 parameters + Dense(439 => 512, relu), # 225_280 parameters + Dense(512 => 256, relu), # 131_328 parameters + Dense(256 => 1), # 257 parameters +) # Total: 10 arrays, 415_648 parameters, 3.172 MiB. + +julia> compressed +Chain( + Dense(2 => 134, relu), # 402 parameters + Dense(134 => 356, relu), # 48_060 parameters + Dense(356 => 464, relu), # 165_648 parameters + Dense(464 => 238, relu), # 110_670 parameters + Dense(238 => 1), # 239 parameters +) # Total: 10 arrays, 325_019 parameters, 2.480 MiB. +``` + +## Conclusions + +In this case, using the LRR bounds is enough to achieve most of the compression in a fraction of the time. + +**Bound tightening and compression must be carefully considered for each individual application.** +- tighter bounds should only be calculated if necessary (e.g. problem cannot be solved quickly enough with loose bounds or the compression has to be maximal) + +*Grimstad and Andresson:* +- integer problems formulated from small to medium NN models (1-3 hidden layers, width of layers >50) are quite fast to solve even with loose bounds + - solving the bounds and then solving the problem itself might be slower than just solving the problem with loose bounds \ No newline at end of file diff --git a/examples/random_nn_comp_test.jl b/examples/random_nn_comp_test.jl index 42558be..ae903b9 100644 --- a/examples/random_nn_comp_test.jl +++ b/examples/random_nn_comp_test.jl @@ -57,17 +57,11 @@ VISUALIZATION """ # compare bounds with/without fast compression -U_data = collect(Iterators.flatten(bounds_U)); -U_true = collect(Iterators.flatten(U_correct)); +plot(collect(Iterators.flatten(bounds_U))) +plot!(collect(Iterators.flatten(U_correct))) -L_data = collect(Iterators.flatten(bounds_L)); -L_true = collect(Iterators.flatten(L_correct)); - -plot(U_data) -plot!(U_true) - -plot!(L_data) -plot!(L_true) +plot!(collect(Iterators.flatten(bounds_L))) +plot!(collect(Iterators.flatten(L_correct))) # compare bounds with/without loose tightening plot(collect(Iterators.flatten(U_correct))) diff --git a/examples/vime_nn_test.jl b/examples/vime_nn_test.jl index 47095ef..c001861 100644 --- a/examples/vime_nn_test.jl +++ b/examples/vime_nn_test.jl @@ -25,10 +25,11 @@ solver_params = SolverParams(solver="Gurobi", silent=true, threads=0, relax=fals # test that created jump model is equal to the original (0.1% tolerance) @test isapprox(vec(model(x)), [forward_pass!(nn_loose, x[:, i])[] for i in 1:size(x)[2]]; rtol=0.001) -plot([forward_pass!(nn_loose, x[:, i])[] for i in 1:size(x)[2]] - vec(model(x))) -plot!(vec(model(x))) - -plot(relu.(collect(Iterators.flatten(U_loose))[1:end-1]) .+ 0.01)#, yscale=:log10) +plot(relu.(collect(Iterators.flatten(U_loose))[1:end-1]) .+ 0.01, + yscale=:log10, + yticks=([1, 100, 500, 1000, 2000], string.([1, 100, 500, 1000, 2000])), + legend=false +) U_data, L_data = get_bounds("/Users/eetureijonen/Desktop/GAMMA-OPT/Gogeta.jl/src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0"); U_data = U_data[3:end]; @@ -68,3 +69,10 @@ compressed, removed = res; # test that compressed model is same as the original @test compressed(x) ≈ model(x) + +# contour plot the model +x_range = LinRange{Float32}(init_L[1], init_U[1], 100); +y_range = LinRange{Float32}(init_L[2], init_U[2], 100); + +contourf(x_range, y_range, (x, y) -> model(hcat(x, y)')[], c=cgrad(:viridis), lw=0) +contourf(x_range, y_range, (x, y) -> compressed(hcat(x, y)')[], c=cgrad(:viridis), lw=0) From 9f3188d53a2ac4e529b1e376566358ed31af8d62 Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Thu, 15 Feb 2024 16:27:58 +0200 Subject: [PATCH 27/32] added no-r bound tightening and added nn tests --- .../layer_0_biases.npy | Bin .../layer_0_weights.npy | Bin .../layer_1_biases.npy | Bin .../layer_1_weights.npy | Bin .../layer_2_biases.npy | Bin .../layer_2_weights.npy | Bin .../layer_3_biases.npy | Bin .../layer_3_weights.npy | Bin .../layer_4_biases.npy | Bin .../layer_4_weights.npy | Bin .../lower_new_lp.npy | Bin .../lower_new_threads.npy | Bin .../neurons_pruned_by_layer.npy | Bin .../model_Adadelta_0.001_0.001_0/results.npy | Bin .../upper_new_lp.npy | Bin .../upper_new_threads.npy | Bin .../pretrained_model_pruner.jl | 0 src/neural_networks/NN_to_MIP.jl | 66 +- src/neural_networks/bounds.jl | 2 +- src/neural_networks/old/JuMP_model.jl | 139 --- src/neural_networks/old/bound_tightening.jl | 881 ------------------ .../old/lossless_compression.jl | 251 ----- test/neural_networks/NN_test.jl | 59 ++ test/neural_networks/{ => old}/CNN_test.jl | 0 .../{ => old}/DNN_bound_tightening_test.jl | 0 test/neural_networks/{ => old}/DNN_test.jl | 0 .../{ => old}/helper_functions.jl | 0 test/runtests.jl | 3 + test/tree_ensembles/TE_test.jl | 2 +- 29 files changed, 118 insertions(+), 1285 deletions(-) rename {src/neural_networks => examples}/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_0_biases.npy (100%) rename {src/neural_networks => examples}/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_0_weights.npy (100%) rename {src/neural_networks => examples}/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_1_biases.npy (100%) rename {src/neural_networks => examples}/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_1_weights.npy (100%) rename {src/neural_networks => examples}/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_2_biases.npy (100%) rename {src/neural_networks => examples}/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_2_weights.npy (100%) rename {src/neural_networks => examples}/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_3_biases.npy (100%) rename {src/neural_networks => examples}/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_3_weights.npy (100%) rename {src/neural_networks => examples}/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_4_biases.npy (100%) rename {src/neural_networks => examples}/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_4_weights.npy (100%) rename {src/neural_networks => examples}/compression/layer_weights/model_Adadelta_0.001_0.001_0/lower_new_lp.npy (100%) rename {src/neural_networks => examples}/compression/layer_weights/model_Adadelta_0.001_0.001_0/lower_new_threads.npy (100%) rename {src/neural_networks => examples}/compression/layer_weights/model_Adadelta_0.001_0.001_0/neurons_pruned_by_layer.npy (100%) rename {src/neural_networks => examples}/compression/layer_weights/model_Adadelta_0.001_0.001_0/results.npy (100%) rename {src/neural_networks => examples}/compression/layer_weights/model_Adadelta_0.001_0.001_0/upper_new_lp.npy (100%) rename {src/neural_networks => examples}/compression/layer_weights/model_Adadelta_0.001_0.001_0/upper_new_threads.npy (100%) rename {src/neural_networks/old => examples}/pretrained_model_pruner.jl (100%) delete mode 100644 src/neural_networks/old/JuMP_model.jl delete mode 100644 src/neural_networks/old/bound_tightening.jl delete mode 100644 src/neural_networks/old/lossless_compression.jl create mode 100644 test/neural_networks/NN_test.jl rename test/neural_networks/{ => old}/CNN_test.jl (100%) rename test/neural_networks/{ => old}/DNN_bound_tightening_test.jl (100%) rename test/neural_networks/{ => old}/DNN_test.jl (100%) rename test/neural_networks/{ => old}/helper_functions.jl (100%) diff --git a/src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_0_biases.npy b/examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_0_biases.npy similarity index 100% rename from src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_0_biases.npy rename to examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_0_biases.npy diff --git a/src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_0_weights.npy b/examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_0_weights.npy similarity index 100% rename from src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_0_weights.npy rename to examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_0_weights.npy diff --git a/src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_1_biases.npy b/examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_1_biases.npy similarity index 100% rename from src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_1_biases.npy rename to examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_1_biases.npy diff --git a/src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_1_weights.npy b/examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_1_weights.npy similarity index 100% rename from src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_1_weights.npy rename to examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_1_weights.npy diff --git a/src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_2_biases.npy b/examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_2_biases.npy similarity index 100% rename from src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_2_biases.npy rename to examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_2_biases.npy diff --git a/src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_2_weights.npy b/examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_2_weights.npy similarity index 100% rename from src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_2_weights.npy rename to examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_2_weights.npy diff --git a/src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_3_biases.npy b/examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_3_biases.npy similarity index 100% rename from src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_3_biases.npy rename to examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_3_biases.npy diff --git a/src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_3_weights.npy b/examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_3_weights.npy similarity index 100% rename from src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_3_weights.npy rename to examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_3_weights.npy diff --git a/src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_4_biases.npy b/examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_4_biases.npy similarity index 100% rename from src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_4_biases.npy rename to examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_4_biases.npy diff --git a/src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_4_weights.npy b/examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_4_weights.npy similarity index 100% rename from src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_4_weights.npy rename to examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/layer_4_weights.npy diff --git a/src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/lower_new_lp.npy b/examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/lower_new_lp.npy similarity index 100% rename from src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/lower_new_lp.npy rename to examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/lower_new_lp.npy diff --git a/src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/lower_new_threads.npy b/examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/lower_new_threads.npy similarity index 100% rename from src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/lower_new_threads.npy rename to examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/lower_new_threads.npy diff --git a/src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/neurons_pruned_by_layer.npy b/examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/neurons_pruned_by_layer.npy similarity index 100% rename from src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/neurons_pruned_by_layer.npy rename to examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/neurons_pruned_by_layer.npy diff --git a/src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/results.npy b/examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/results.npy similarity index 100% rename from src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/results.npy rename to examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/results.npy diff --git a/src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/upper_new_lp.npy b/examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/upper_new_lp.npy similarity index 100% rename from src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/upper_new_lp.npy rename to examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/upper_new_lp.npy diff --git a/src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/upper_new_threads.npy b/examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/upper_new_threads.npy similarity index 100% rename from src/neural_networks/compression/layer_weights/model_Adadelta_0.001_0.001_0/upper_new_threads.npy rename to examples/compression/layer_weights/model_Adadelta_0.001_0.001_0/upper_new_threads.npy diff --git a/src/neural_networks/old/pretrained_model_pruner.jl b/examples/pretrained_model_pruner.jl similarity index 100% rename from src/neural_networks/old/pretrained_model_pruner.jl rename to examples/pretrained_model_pruner.jl diff --git a/src/neural_networks/NN_to_MIP.jl b/src/neural_networks/NN_to_MIP.jl index 4664853..ed0036d 100644 --- a/src/neural_networks/NN_to_MIP.jl +++ b/src/neural_networks/NN_to_MIP.jl @@ -1,3 +1,8 @@ +""" + SolverParams() + +Parameters to be used by the solver. +""" @kwdef struct SolverParams solver::String silent::Bool @@ -6,10 +11,13 @@ time_limit::Float64 end -function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}, solver_params::SolverParams; tighten_bounds::Bool=false) +function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}, solver_params::SolverParams; tighten_bounds::String="fast", bounds_U=nothing, bounds_L=nothing, out_ub=nothing, out_lb=nothing) + + precomputed_bounds = (bounds_U !== nothing) && (bounds_L !== nothing) + @assert tighten_bounds in ("fast", "standard", "output") K = length(NN_model) # number of layers (input layer not included) - @assert reduce(&, [NN_model[i].σ == relu for i in 1:K-1]) "Neural network must use the relu activation function." + @assert all([NN_model[i].σ == relu for i in 1:K-1]) "Neural network must use the relu activation function." @assert NN_model[K].σ == identity "Neural network must use the identity function for the output layer." W = [Flux.params(NN_model)[2*k-1] for k in 1:K] @@ -20,6 +28,7 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect neurons(layer) = layer == 0 ? [i for i in 1:input_length] : [i for i in 1:neuron_count[layer]] @assert input_length == length(init_ub) == length(init_lb) "Initial bounds arrays must be the same length as the input layer" + if precomputed_bounds @assert length.(bounds_U) == length.([neuron_count[layer] for layer in 1:K]) end # build model up to second layer jump_model = Model() @@ -32,21 +41,18 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect @constraint(jump_model, [j = 1:input_length], x[0, j] <= init_ub[j]) @constraint(jump_model, [j = 1:input_length], x[0, j] >= init_lb[j]) - bounds_U = Vector{Vector}(undef, K) - bounds_L = Vector{Vector}(undef, K) + if precomputed_bounds == false + bounds_U = Vector{Vector}(undef, K) + bounds_L = Vector{Vector}(undef, K) + end for layer in 1:K # hidden layers and output println("\nLAYER $layer") - if tighten_bounds - bounds = if nprocs() > 1 - pmap(neuron -> calculate_bounds(copy_model(jump_model, solver_params), layer, neuron, W, b, neurons), neurons(layer)) - else - map(neuron -> calculate_bounds(jump_model, layer, neuron, W, b, neurons), neurons(layer)) - end - bounds_U[layer], bounds_L[layer] = [bound[1] for bound in bounds], [bound[2] for bound in bounds] - else + if precomputed_bounds == false + + # compute loose bounds if layer == 1 bounds_U[layer] = [sum(max(W[layer][neuron, previous] * init_ub[previous], W[layer][neuron, previous] * init_lb[previous]) for previous in neurons(layer-1)) + b[layer][neuron] for neuron in neurons(layer)] bounds_L[layer] = [sum(min(W[layer][neuron, previous] * init_ub[previous], W[layer][neuron, previous] * init_lb[previous]) for previous in neurons(layer-1)) + b[layer][neuron] for neuron in neurons(layer)] @@ -54,6 +60,18 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect bounds_U[layer] = [sum(max(W[layer][neuron, previous] * max(0, bounds_U[layer-1][previous]), W[layer][neuron, previous] * max(0, bounds_L[layer-1][previous])) for previous in neurons(layer-1)) + b[layer][neuron] for neuron in neurons(layer)] bounds_L[layer] = [sum(min(W[layer][neuron, previous] * max(0, bounds_U[layer-1][previous]), W[layer][neuron, previous] * max(0, bounds_L[layer-1][previous])) for previous in neurons(layer-1)) + b[layer][neuron] for neuron in neurons(layer)] end + + if tighten_bounds == "standard" + bounds = if nprocs() > 1 + pmap(neuron -> calculate_bounds(copy_model(jump_model, solver_params), layer, neuron, W, b, neurons), neurons(layer)) + else + map(neuron -> calculate_bounds(jump_model, layer, neuron, W, b, neurons), neurons(layer)) + end + + # only change if bound is improved + bounds_U[layer] = min.(bounds_U[layer], [bound[1] for bound in bounds]) + bounds_L[layer] = max.(bounds_L[layer], [bound[2] for bound in bounds]) + end end if layer == K # output bounds calculated but no unnecessary constraints added @@ -77,6 +95,30 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect # output layer @constraint(jump_model, [neuron in 1:neuron_count[K]], x[K, neuron] == b[K][neuron] + sum(W[K][neuron, i] * x[K-1, i] for i in neurons(K-1))) + # using output bounds in bound tightening + if tighten_bounds == "output" + @assert length(out_lb) == length(out_ub) == neuron_count[K] "Incorrect length of output bounds array." + + @constraint(jump_model, [neuron in 1:neuron_count[K]], x[K, neuron] >= out_lb[neuron]) + @constraint(jump_model, [neuron in 1:neuron_count[K]], x[K, neuron] <= out_ub[neuron]) + + for layer in 1:K-1, neuron in neuron_count[layer] + bounds = if nprocs() > 1 + pmap(neuron -> calculate_bounds(copy_model(jump_model, solver_params), layer, neuron, W, b, neurons), neurons(layer)) + else + map(neuron -> calculate_bounds(jump_model, layer, neuron, W, b, neurons), neurons(layer)) + end + + # only change if bound is improved + bounds_U[layer] = min.(bounds_U[layer], [bound[1] for bound in bounds]) + bounds_L[layer] = max.(bounds_L[layer], [bound[2] for bound in bounds]) + + @constraint(jump_model, x[layer, neuron] <= max(0, bounds_U[layer][neuron]) * (1 - z[layer, neuron])) + @constraint(jump_model, s[layer, neuron] <= max(0, -bounds_L[layer][neuron]) * z[layer, neuron]) + + end + end + return jump_model, bounds_U, bounds_L end diff --git a/src/neural_networks/bounds.jl b/src/neural_networks/bounds.jl index 4ee0ea3..e6a6b7d 100644 --- a/src/neural_networks/bounds.jl +++ b/src/neural_networks/bounds.jl @@ -46,7 +46,7 @@ function calculate_bounds(model::JuMP.Model, layer, neuron, W, b, neurons; layer if upper_bound > 1_000 @warn "Upper bound is very loose: $upper_bound, problem might become infeasible." end if lower_bound < -1_000 @warn "Lower bound is very loose: $lower_bound, problem might become infeasible." end - println("Neuron: $neuron") + println("Neuron: $neuron [$lower_bound, $upper_bound]") return upper_bound, lower_bound end diff --git a/src/neural_networks/old/JuMP_model.jl b/src/neural_networks/old/JuMP_model.jl deleted file mode 100644 index 8ebd1bf..0000000 --- a/src/neural_networks/old/JuMP_model.jl +++ /dev/null @@ -1,139 +0,0 @@ -using JuMP, Flux, Gurobi -using Flux: params - -""" - create_JuMP_model(DNN::Chain, L_bounds::Vector{Float32}, U_bounds::Vector{Float32}, bound_tightening::String="none", bt_verbose::Bool=false) - -Converts a ReLU DNN to a 0-1 MILP JuMP model. The ReLU DNN is assumed to be a Flux.Chain. -The activation function must be "relu" in all hidden layers and "identity" in the output layer. -The lower and upper bounds to the function are given as a Vector{Float32}. The bounds are in order from the input layer to the output layer. -The keyword argument "bt" determines if bound tightening is to be used on the constraint bounds: -— "none": No bound tightening is used. -— "singletread": One shared JuMP model is used to calculate bounds one at a time. -— "threads": The bounds are calculated using a separate model for each subproblem using Threads -— "workers": The bounds are calculated using a separate model for each subproblem using Workers -— "2 workers": The bounds are calculated with a maximum of two workers at each layer (upper and lower bounds). Each worker reuses the JuMP model. - -# Arguments -- `DNN::Chain`: A trained ReLU DNN. -- `L_bounds::Vector{Float32}`: Lower bounds on the node values of the DNN. -- `U_bounds::Vector{Float32}`: Upper bounds on the node values of the DNN. -- `bt::String="none"`: Optional bound tightening of the constraint bounds. Can be set to "none", "singlethread", "threads", "workers" or "2 workers". -- `bt_verbose::Bool=false`: Controls Gurobi logs in bound tightening procedures. - -# Examples -```julia -model = create_JuMP_model(DNN, L_bounds, U_bounds, "singlethread", false) -``` -""" -function create_JuMP_model(DNN::Chain, L_bounds::Vector{Float32}, U_bounds::Vector{Float32}, bt::String="none", bt_verbose::Bool=false) - - K = length(DNN) # NOTE! there are K+1 layers in the nn - for i in 1:K-1 - @assert DNN[i].σ == relu "Hidden layers must use \"relu\" as the activation function" - end - @assert DNN[K].σ == identity "Output layer must use the \"identity\" activation function" - - # store the DNN weights and biases - DNN_params = Flux.params(DNN) - W = [DNN_params[2*i-1] for i in 1:K] - b = [DNN_params[2*i] for i in 1:K] - - # stores the node count of layer k (starting at layer k=0) at index k+1 - input_node_count = length(DNN_params[1][1, :]) - node_count = [if k == 1 input_node_count else length(DNN_params[2*(k-1)]) end for k in 1:K+1] - - final_L_bounds = copy(L_bounds) - final_U_bounds = copy(U_bounds) - - # optional: calculates optimal lower and upper bounds L and U - @assert bt == "none" || bt == "singlethread" || bt == "threads" || bt == "workers" || bt == "2 workers" - "bound_tightening has to be set to \"none\", \"singlethread\", \"threads\", \"workers\" or \"2 workers\"." - if bt == "singlethread" - final_L_bounds, final_U_bounds = bound_tightening(DNN, U_bounds, L_bounds, bt_verbose) - elseif bt == "threads" - final_L_bounds, final_U_bounds = bound_tightening_threads(DNN, U_bounds, L_bounds, bt_verbose) - elseif bt == "workers" - final_L_bounds, final_U_bounds = bound_tightening_workers(DNN, U_bounds, L_bounds, bt_verbose) - elseif bt == "2 workers" - final_L_bounds, final_U_bounds = bound_tightening_2workers(DNN, U_bounds, L_bounds, bt_verbose) - end - - model = Model(optimizer_with_attributes(Gurobi.Optimizer)) - - # sets the variables x[k,j] and s[k,j], the binary variables z[k,j] and the big-M values U[k,j] and L[k,j] - @variable(model, x[k in 0:K, j in 1:node_count[k+1]] >= 0) - if K > 1 # s and z variables only to hidden layers, i.e., layers 1:K-1 - @variable(model, s[k in 1:K, j in 1:node_count[k+1]] >= 0) - @variable(model, z[k in 1:K, j in 1:node_count[k+1]], Bin) - end - @variable(model, U[k in 0:K, j in 1:node_count[k+1]]) - @variable(model, L[k in 0:K, j in 1:node_count[k+1]]) - - # fix values to all U[k,j] and L[k,j] from U_bounds and L_bounds - index = 1 - for k in 0:K - for j in 1:node_count[k+1] - fix(U[k, j], final_U_bounds[index]) - fix(L[k, j], final_L_bounds[index]) - index += 1 - end - end - - # fix bounds U and L to input nodes - for input_node in 1:node_count[1] - delete_lower_bound(x[0, input_node]) - @constraint(model, L[0, input_node] <= x[0, input_node]) - @constraint(model, x[0, input_node] <= U[0, input_node]) - end - - # constraints corresponding to the ReLU activation functions - for k in 1:K - for node in 1:node_count[k+1] # node count of the next layer of k, i.e., the layer k+1 - temp_sum = sum(W[k][node, j] * x[k-1, j] for j in 1:node_count[k]) - if k < K # hidden layers: k = 1, ..., K-1 - @constraint(model, temp_sum + b[k][node] == x[k, node] - s[k, node]) - else # output layer: k == K - @constraint(model, temp_sum + b[k][node] == x[k, node]) - end - end - end - - # fix bounds to the hidden layer nodes - @constraint(model, [k in 1:K, j in 1:node_count[k+1]], x[k, j] <= U[k, j] * z[k, j]) - @constraint(model, [k in 1:K, j in 1:node_count[k+1]], s[k, j] <= -L[k, j] * (1 - z[k, j])) - - # fix bounds to the output nodes - for output_node in 1:node_count[K+1] - delete_lower_bound(x[K, output_node]) - @constraint(model, L[K, output_node] <= x[K, output_node]) - @constraint(model, x[K, output_node] <= U[K, output_node]) - end - - @objective(model, Max, x[1, 1]) # arbitrary objective function to have a complete JuMP model - - return model -end - -""" - evaluate!(JuMP_model::Model, input::Vector{Float32}) - -Fixes the variables corresponding to the DNN input to a given input vector. - -# Arguments -- `JuMP_model::Model`: A JuMP model representing a traied ReLU DNN (generated using the function create_JuMP_model). -- `input::Vector{Float32}`: A given input to the trained DNN. - -# Examples -```julia -evaluate!(JuMP_model, input) -``` -""" -function evaluate!(JuMP_model::Model, input::Vector{Float32}) - x = JuMP_model[:x] # stores the @variable with name x from the JuMP_model - input_len = length(input) - @assert input_len == length(x[0, :]) "\"input\" has wrong dimension" - for input_node in 1:input_len - fix(x[0, input_node], input[input_node], force=true) # fix value of input to x[0,j] - end -end diff --git a/src/neural_networks/old/bound_tightening.jl b/src/neural_networks/old/bound_tightening.jl deleted file mode 100644 index 9f917be..0000000 --- a/src/neural_networks/old/bound_tightening.jl +++ /dev/null @@ -1,881 +0,0 @@ -using JuMP, Flux, Gurobi -using JuMP: Model -using Flux: params -using Distributed -using SharedArrays - -""" -bound_tightening(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false) - -A single-threaded implementation of optimal tightened constraint bounds L and U for for a trained DNN. -Using these bounds with the create_JuMP_model function reduces solution time for optimization problems. - -# Arguments -- `DNN::Chain`: A trained ReLU DNN. -- `init_U_bounds::Vector{Float32}`: Initial upper bounds on the node values of the DNN. -- `init_L_bounds::Vector{Float32}`: Initial lower bounds on the node values of the DNN. -- `verbose::Bool=false`: Controls Gurobi logs. - -# Examples -```julia -L_bounds, U_bounds = bound_tightening(DNN, init_U_bounds, init_L_bounds, false) -``` -""" - -function bound_tightening(DNN::Chain, init_U_bounds::Vector{Float64}, init_L_bounds::Vector{Float64}, verbose::Bool=false, lp_relaxation::Bool=false, gurobi_env=Gurobi.Env()) - - K = length(DNN) # NOTE! there are K+1 layers in the nn - - # store the DNN weights and biases - DNN_params = params(DNN) - W = [DNN_params[2*i-1] for i in 1:K] - b = [DNN_params[2*i] for i in 1:K] - - # stores the node count of layer k (starting at layer k=0) at index k+1 - input_node_count = length(DNN_params[1][1, :]) - node_count = [if k == 1 input_node_count else length(DNN_params[2*(k-1)]) end for k in 1:K+1] - - # store the current optimal bounds in the algorithm - curr_U_bounds = copy(init_U_bounds) - curr_L_bounds = copy(init_L_bounds) - - model = Model(optimizer_with_attributes(Gurobi.Optimizer, "OutputFlag" => (verbose ? 1 : 0))) - - # keeps track of the current node index starting from layer 1 (out of 0:K) - outer_index = node_count[1] + 1 - - # NOTE! below variables and constraints for all opt problems - @variable(model, x[k in 0:K, j in 1:node_count[k+1]] >= 0) - @variable(model, s[k in 1:K-1, j in 1:node_count[k+1]] >= 0) - @variable(model, z[k in 1:K-1, j in 1:node_count[k+1]], Bin) - @variable(model, U[k in 0:K, j in 1:node_count[k+1]]) - @variable(model, L[k in 0:K, j in 1:node_count[k+1]]) - - # fix values to all U[k,j] and L[k,j] from U_bounds and L_bounds - index = 1 - for k in 0:K - for j in 1:node_count[k+1] - fix(U[k, j], curr_U_bounds[index], force=true) - fix(L[k, j], curr_L_bounds[index], force=true) - index += 1 - end - end - - # input layer (layer 0) node bounds are given beforehand - for input_node in 1:node_count[1] - delete_lower_bound(x[0, input_node]) - @constraint(model, L[0, input_node] <= x[0, input_node]) - @constraint(model, x[0, input_node] <= U[0, input_node]) - end - - # deleting lower bound for output nodes - for output_node in 1:node_count[K+1] - delete_lower_bound(x[K, output_node]) - end - - # NOTE! below constraints depending on the layer - for k in 1:K - println() - println("Working on layer $k.") - # we only want to build ALL of the constraints until the PREVIOUS layer, and then go node by node - # here we calculate ONLY the constraints until the PREVIOUS layer - for node_in in 1:node_count[k] - if k >= 2 - temp_sum = AffExpr() - for j in 1:node_count[k-1] - add_to_expression!(temp_sum, W[k-1][node_in, j], x[k-1-1, j]) - end - # temp_sum = sum(W[k-1][node_in, j] * x[k-1-1, j] for j in 1:node_count[k-1]) - @constraint(model, x[k-1, node_in] <= U[k-1, node_in] * z[k-1, node_in]) - @constraint(model, s[k-1, node_in] <= -L[k-1, node_in] * (1 - z[k-1, node_in])) - if k <= K - 1 - @constraint(model, temp_sum + b[k-1][node_in] == x[k-1, node_in] - s[k-1, node_in]) - else # k == K - @constraint(model, temp_sum + b[k-1][node_in] == x[k-1, node_in]) - end - end - end - - # NOTE! below constraints depending on the node - for node in 1:node_count[k+1] - # here we calculate the specific constraints depending on the current node - temp_sum = AffExpr() - for j in 1:node_count[k] - add_to_expression!(temp_sum, W[k][node, j], x[k-1, j]) - end - if k <= K - 1 - @constraint(model, node_con, temp_sum + b[k][node] == x[k, node] - s[k, node]) - @constraint(model, node_U, x[k, node] <= U[k, node] * z[k, node]) - @constraint(model, node_L, s[k, node] <= -L[k, node] * (1 - z[k, node])) - elseif k == K # == last value of k - @constraint(model, node_con, temp_sum + b[k][node] == x[k, node]) - @constraint(model, node_L, L[k, node] <= x[k, node]) - @constraint(model, node_U, x[k, node] <= U[k, node]) - end - - # NOTE! below objective function and optimizing the model depending on obj_function and layer - for obj_function in 1:2 - if obj_function == 1 && k <= K - 1 # Min, hidden layer - @objective(model, Min, x[k, node] - s[k, node]) - elseif obj_function == 2 && k <= K - 1 # Max, hidden layer - @objective(model, Max, x[k, node] - s[k, node]) - elseif obj_function == 1 && k == K # Min, last layer - @objective(model, Min, x[k, node]) - elseif obj_function == 2 && k == K # Max, last layer - @objective(model, Max, x[k, node]) - end - - if lp_relaxation - undo = relax_integrality(model) - set_optimizer(model, () -> Gurobi.Optimizer(gurobi_env)) - set_silent(model) - end - - solve_time = @elapsed optimize!(model) - solve_time = round(solve_time; sigdigits = 3) - - optimal = nothing - - try - optimal = objective_value(model) - catch - optimal = obj_function == 1 ? curr_L_bounds[outer_index] : curr_U_bounds[outer_index] - end - - print(".") - - # fix the model variable L or U corresponding to the current node to be the optimal value - if obj_function == 1 # Min - curr_L_bounds[outer_index] = optimal - fix(L[k, node], optimal) - elseif obj_function == 2 # Max - curr_U_bounds[outer_index] = optimal - fix(U[k, node], optimal) - end - end - outer_index += 1 - - # deleting and unregistering the constraints assigned to the current node - delete(model, node_con) - delete(model, node_L) - delete(model, node_U) - unregister(model, :node_con) - unregister(model, :node_L) - unregister(model, :node_U) - end - end - - println("Solving optimal constraint bounds single-threaded complete") - - return curr_U_bounds, curr_L_bounds -end - -function bound_tightening_threads_old(DNN::Chain, init_U_bounds::Vector{Float64}, init_L_bounds::Vector{Float64}, verbose::Bool=false) - - K = length(DNN) # NOTE! there are K+1 layers in the nn - - # store the DNN weights and biases - DNN_params = params(DNN) - W = [DNN_params[2*i-1] for i in 1:K] - b = [DNN_params[2*i] for i in 1:K] - - # stores the node count of layer k (starting at layer k=0) at index k+1 - input_node_count = length(DNN_params[1][1, :]) - node_count = [if k == 1 input_node_count else length(DNN_params[2*(k-1)]) end for k in 1:K+1] - - # store the current optimal bounds in the algorithm - curr_U_bounds = copy(init_U_bounds) - curr_L_bounds = copy(init_L_bounds) - - lock = Threads.ReentrantLock() - - for k in 1:2 - - GC.gc() - - println() - println("Working on layer $k.") - - Threads.@threads for node in 1:(2*node_count[k+1]) # loop over both obj functions - - ### below variables and constraints in all problems - - model = Model(optimizer_with_attributes(() -> Gurobi.Optimizer(gurobi_env))) - # set_optimizer_attribute(model, "MIPGap", 0.1) - set_silent(model) - - # model = Model(optimizer_with_attributes(Gurobi.Optimizer, "OutputFlag" => (verbose ? 1 : 0))) - - # keeps track of the current node index starting from layer 1 (out of 0:K) - prev_layers_node_sum = 0 - for prev_layer in 0:k-1 - prev_layers_node_sum += node_count[prev_layer+1] - end - - # loops nodes twice: 1st time with obj function Min, 2nd time with Max - curr_node = node - obj_function = 1 - if node > node_count[k+1] - curr_node = node - node_count[k+1] - obj_function = 2 - end - curr_node_index = prev_layers_node_sum + curr_node - - # NOTE! below variables and constraints for all opt problems - @variable(model, x[k in 0:K, j in 1:node_count[k+1]] >= 0) - @variable(model, s[k in 1:K-1, j in 1:node_count[k+1]] >= 0) - @variable(model, z[k in 1:K-1, j in 1:node_count[k+1]], Bin) - @variable(model, U[k in 0:K, j in 1:node_count[k+1]]) - @variable(model, L[k in 0:K, j in 1:node_count[k+1]]) - - # fix values to all U[k,j] and L[k,j] from U_bounds and L_bounds - index = 1 - Threads.lock(lock) do - for k in 0:K - for j in 1:node_count[k+1] - fix(U[k, j], curr_U_bounds[index], force=true) - fix(L[k, j], curr_L_bounds[index], force=true) - index += 1 - end - end - end - - # input layer (layer 0) node bounds are given beforehand - for input_node in 1:node_count[1] - delete_lower_bound(x[0, input_node]) - @constraint(model, L[0, input_node] <= x[0, input_node]) - @constraint(model, x[0, input_node] <= U[0, input_node]) - end - - # deleting lower bound for output nodes - for output_node in 1:node_count[K+1] - delete_lower_bound(x[K, output_node]) - end - - ### below constraints depending on the layer (every constraint up to the previous layer) - for k_in in 1:k - for node_in in 1:node_count[k_in] - if k_in >= 2 - temp_sum = sum(W[k_in-1][node_in, j] * x[k_in-1-1, j] for j in 1:node_count[k_in-1]) - @constraint(model, x[k_in-1, node_in] <= U[k_in-1, node_in] * z[k_in-1, node_in]) - @constraint(model, s[k_in-1, node_in] <= -L[k_in-1, node_in] * (1 - z[k_in-1, node_in])) - if k_in <= K - 1 - @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in] - s[k_in-1, node_in]) - else # k_in == K - @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in]) - end - end - end - end - - ### below constraints depending on the node - temp_sum = sum(W[k][curr_node, j] * x[k-1, j] for j in 1:node_count[k]) # NOTE! prev layer [k] - if k <= K - 1 - @constraint(model, node_con, temp_sum + b[k][curr_node] == x[k, curr_node] - s[k, curr_node]) - @constraint(model, node_U, x[k, curr_node] <= U[k, curr_node] * z[k, curr_node]) - @constraint(model, node_L, s[k, curr_node] <= -L[k, curr_node] * (1 - z[k, curr_node])) - elseif k == K # == last value of k - @constraint(model, node_con, temp_sum + b[k][curr_node] == x[k, curr_node]) - @constraint(model, node_L, L[k, curr_node] <= x[k, curr_node]) - @constraint(model, node_U, x[k, curr_node] <= U[k, curr_node]) - end - - if obj_function == 1 && k <= K - 1 # Min, hidden layer - @objective(model, Min, x[k, curr_node] - s[k, curr_node]) - elseif obj_function == 2 && k <= K - 1 # Max, hidden layer - @objective(model, Max, x[k, curr_node] - s[k, curr_node]) - elseif obj_function == 1 && k == K # Min, last layer - @objective(model, Min, x[k, curr_node]) - elseif obj_function == 2 && k == K # Max, last layer - @objective(model, Max, x[k, curr_node]) - end - - solve_time = @elapsed optimize!(model) - solve_time = round(solve_time; sigdigits = 3) - - optimal = nothing - # @assert termination_status(model) == OPTIMAL || termination_status(model) == TIME_LIMIT - # "Problem (layer $k (from 1:$K), node $curr_node, $(obj_function == 1 ? "L" : "U")-bound) is infeasible." - try - optimal = objective_value(model) - catch - optimal = obj_function == 1 ? curr_L_bounds[curr_node_index] : curr_U_bounds[curr_node_index] - end - - print(".") - - # fix the model variable L or U corresponding to the current node to be the optimal value - Threads.lock(lock) do - if obj_function == 1 # Min - curr_L_bounds[curr_node_index] = optimal - fix(L[k, curr_node], optimal) - elseif obj_function == 2 # Max - curr_U_bounds[curr_node_index] = optimal - fix(U[k, curr_node], optimal) - end - end - - model = nothing - - end - - end - - println("Solving optimal constraint bounds using threads complete") - - return curr_U_bounds, curr_L_bounds -end - -""" -bound_tightening_threads(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false) - -A multi-threaded (using Threads) implementation of optimal tightened constraint bounds L and U for for a trained DNN. -Using these bounds with the create_JuMP_model function reduces solution time for optimization problems. - -# Arguments -- `DNN::Chain`: A trained ReLU DNN. -- `init_U_bounds::Vector{Float32}`: Initial upper bounds on the node values of the DNN. -- `init_L_bounds::Vector{Float32}`: Initial lower bounds on the node values of the DNN. -- `verbose::Bool=false`: Controls Gurobi logs. - -# Examples -```julia -L_bounds_threads, U_bounds_threads = bound_tightening_threads(DNN, init_U_bounds, init_L_bounds, false) -``` -""" - -function bound_tightening_threads(DNN::Chain, init_U_bounds::Vector{Float64}, init_L_bounds::Vector{Float64}, verbose::Bool=false, gurobi_env=Gurobi.Env()) - - K = length(DNN) # NOTE! there are K+1 layers in the nn - - # store the DNN weights and biases - DNN_params = params(DNN) - W = [DNN_params[2*i-1] for i in 1:K] - b = [DNN_params[2*i] for i in 1:K] - - # stores the node count of layer k (starting at layer k=0) at index k+1 - input_node_count = length(DNN_params[1][1, :]) - node_count = [if k == 1 input_node_count else length(DNN_params[2*(k-1)]) end for k in 1:K+1] - - # store the current optimal bounds in the algorithm - curr_U_bounds = copy(init_U_bounds) - curr_L_bounds = copy(init_L_bounds) - - lock = Threads.ReentrantLock() - - for k in 1:K - prev_layers_node_sum = sum(node_count[1:k]) - - println() - println("Working on layer $k.") - - GC.gc() - - Threads.@threads for node in 1:(2*node_count[k+1]) # loop over both obj functions - - optimal, obj_function, curr_node_index = bound_tightening_threads_inner( - k, - K, - node, - node_count, - prev_layers_node_sum, - curr_U_bounds, - curr_L_bounds, - W, - b, - verbose, - gurobi_env - ) - - # fix the model variable L or U corresponding to the current node to be the optimal value - Threads.lock(lock) do - if obj_function == 1 # Min - curr_L_bounds[curr_node_index] = optimal - # fix(L[k, curr_node], optimal) - elseif obj_function == 2 # Max - curr_U_bounds[curr_node_index] = optimal - # fix(U[k, curr_node], optimal) - end - end - - end - - end - - println("Solving optimal constraint bounds using threads complete") - - return curr_U_bounds, curr_L_bounds -end - -function bound_tightening_threads_inner(k, K, node, node_count, prev_layers_node_sum, current_upper_bounds, current_lower_bounds, W, b, verbose, gurobi_env) - creation_time = @elapsed begin - - model = Model(optimizer_with_attributes(() -> Gurobi.Optimizer(gurobi_env))) - # model attributes - set_attribute(model, "OutputFlag", (verbose ? 1 : 0)) - set_attribute(model, "TimeLimit", 10) - # set_optimizer_attribute(model, "MIPGap", 0.1) - set_silent(model) - - # loops nodes twice: 1st time with obj function Min, 2nd time with Max - curr_node = node - obj_function = 1 - if node > node_count[k + 1] - curr_node = node - node_count[k + 1] - obj_function = 2 - end - curr_node_index = prev_layers_node_sum + curr_node - - # NOTE! below variables and constraints for all opt problems - @variable(model, x[k in 0:K, j in 1:node_count[k+1]] >= 0) - @variable(model, s[k in 1:K-1, j in 1:node_count[k+1]] >= 0) - @variable(model, z[k in 1:K-1, j in 1:node_count[k+1]], Bin) - @variable(model, U[k in 0:K, j in 1:node_count[k+1]]) - @variable(model, L[k in 0:K, j in 1:node_count[k+1]]) - - - # fix values to all U[k,j] and L[k,j] from U_bounds and L_bounds - index = 1 - for k in 0:K - for j in 1:node_count[k + 1] - fix(U[k, j], current_upper_bounds[index], force=true) - fix(L[k, j], current_lower_bounds[index], force=true) - index += 1 - end - end - - # input layer (layer 0) node bounds are given beforehand - for input_node in 1:node_count[1] - delete_lower_bound(x[0, input_node]) - @constraint(model, L[0, input_node] <= x[0, input_node]) - @constraint(model, x[0, input_node] <= U[0, input_node]) - end - - # deleting lower bound for output nodes - for output_node in 1:node_count[K+1] - delete_lower_bound(x[K, output_node]) - end - - ### below constraints depending on the layer (every constraint up to the previous layer) - for k_in in 1:k - for node_in in 1:node_count[k_in] - if k_in >= 2 - temp_sum = AffExpr() - for j in 1:node_count[k_in-1] - add_to_expression!(temp_sum, W[k_in-1][node_in, j], x[k_in - 2, j]) - end - # temp_sum = sum(W[k_in-1][node_in, j] * x[k_in-1-1, j] for j in 1:node_count[k_in-1]) - @constraint(model, x[k_in-1, node_in] <= U[k_in-1, node_in] * z[k_in-1, node_in]) - @constraint(model, s[k_in-1, node_in] <= -L[k_in-1, node_in] * (1 - z[k_in-1, node_in])) - if k_in <= K - 1 - @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in] - s[k_in-1, node_in]) - else # k_in == K - @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in]) - end - end - end - end - - ### below constraints depending on the node - temp_sum = AffExpr() - for j in 1:node_count[k] - add_to_expression!(temp_sum, W[k][curr_node, j], x[k - 1, j]) - end - - if k <= K - 1 - @constraint(model, node_con, temp_sum + b[k][curr_node] == x[k, curr_node] - s[k, curr_node]) - @constraint(model, node_U, x[k, curr_node] <= U[k, curr_node] * z[k, curr_node]) - @constraint(model, node_L, s[k, curr_node] <= -L[k, curr_node] * (1 - z[k, curr_node])) - elseif k == K # == last value of k - @constraint(model, node_con, temp_sum + b[k][curr_node] == x[k, curr_node]) - @constraint(model, node_L, L[k, curr_node] <= x[k, curr_node]) - @constraint(model, node_U, x[k, curr_node] <= U[k, curr_node]) - end - - if obj_function == 1 && k <= K - 1 # Min, hidden layer - @objective(model, Min, x[k, curr_node] - s[k, curr_node]) - elseif obj_function == 2 && k <= K - 1 # Max, hidden layer - @objective(model, Max, x[k, curr_node] - s[k, curr_node]) - elseif obj_function == 1 && k == K # Min, last layer - @objective(model, Min, x[k, curr_node]) - elseif obj_function == 2 && k == K # Max, last layer - @objective(model, Max, x[k, curr_node]) - end - - end - - # println("TIME SPENT CREATING MODEL: $(round(creation_time, digits=2)) seconds") - - solve_time = @elapsed optimize!(model) - solve_time = round(solve_time; sigdigits = 3) - - optimal = nothing - try - optimal = objective_value(model) - catch - optimal = obj_function == 1 ? current_lower_bounds[curr_node_index] : current_upper_bounds[curr_node_index] - end - - print(".") - - # set to nothing to speed up garbage collection - model = nothing - - return optimal, obj_function, curr_node_index -end - -""" -bound_tightening_workers(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false) - -A multi-threaded (using workers) implementation of optimal tightened constraint bounds L and U for for a trained DNN. -Using these bounds with the create_JuMP_model function reduces solution time for optimization problems. - -# Arguments -- `DNN::Chain`: A trained ReLU DNN. -- `init_U_bounds::Vector{Float32}`: Initial upper bounds on the node values of the DNN. -- `init_L_bounds::Vector{Float32}`: Initial lower bounds on the node values of the DNN. -- `verbose::Bool=false`: Controls Gurobi logs. - -# Examples -```julia -L_bounds_workers, U_bounds_workers = bound_tightening_workers(DNN, init_U_bounds, init_L_bounds, false) -``` -""" - -function bound_tightening_workers(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false) - - K = length(DNN) # NOTE! there are K+1 layers in the nn - - # store the DNN weights and biases - DNN_params = params(DNN) - W = [DNN_params[2*i-1] for i in 1:K] - b = [DNN_params[2*i] for i in 1:K] - - # stores the node count of layer k (starting at layer k=0) at index k+1 - input_node_count = length(DNN_params[1][1, :]) - node_count = [if k == 1 input_node_count else length(DNN_params[2*(k-1)]) end for k in 1:K+1] - - # store the current optimal bounds in the algorithm - curr_U_bounds = copy(init_U_bounds) - curr_L_bounds = copy(init_L_bounds) - - for k in 1:K - - # Distributed.pmap returns the bounds in order - L_U_bounds = Distributed.pmap(node -> bt_workers_inner(K, k, node, W, b, node_count, curr_U_bounds, curr_L_bounds, verbose), 1:(2*node_count[k+1])) - - for node in 1:node_count[k+1] - prev_layers_node_sum = 0 - for prev_layer in 0:k-1 - prev_layers_node_sum += node_count[prev_layer+1] - end - - # loops nodes twice: 1st time with obj function Min, 2nd time with Max - curr_node = node - obj_function = 1 - if node > node_count[k+1] - curr_node = node - node_count[k+1] - obj_function = 2 - end - curr_node_index = prev_layers_node_sum + curr_node - - # L-bounds in 1:node_count[k+1], U-bounds in 1:(node + node_count[k+1]) - curr_L_bounds[curr_node_index] = L_U_bounds[node] - curr_U_bounds[curr_node_index] = L_U_bounds[node + node_count[k+1]] - end - - end - - println("Solving optimal constraint bounds using workers complete") - - return curr_U_bounds, curr_L_bounds -end - -# Inner function to bound_tightening_workers: assigns a JuMP model to the current worker - -function bt_workers_inner( - K::Int64, - k::Int64, - node::Int64, - W::Vector{Matrix{Float32}}, - b::Vector{Vector{Float32}}, - node_count::Vector{Int64}, - curr_U_bounds::Vector{Float32}, - curr_L_bounds::Vector{Float32}, - verbose::Bool - ) - - model = Model(optimizer_with_attributes(Gurobi.Optimizer, "OutputFlag" => (verbose ? 1 : 0), "Threads" => 1)) - - # keeps track of the current node index starting from layer 1 (out of 0:K) - prev_layers_node_sum = 0 - for prev_layer in 0:k-1 - prev_layers_node_sum += node_count[prev_layer+1] - end - - # loops nodes twice: 1st time with obj function Min, 2nd time with Max - curr_node = node - obj_function = 1 - if node > node_count[k+1] - curr_node = node - node_count[k+1] - obj_function = 2 - end - - # NOTE! below variables and constraints for all opt problems - @variable(model, x[k in 0:K, j in 1:node_count[k+1]] >= 0) - @variable(model, s[k in 1:K-1, j in 1:node_count[k+1]] >= 0) - @variable(model, z[k in 1:K-1, j in 1:node_count[k+1]], Bin) - @variable(model, U[k in 0:K, j in 1:node_count[k+1]]) - @variable(model, L[k in 0:K, j in 1:node_count[k+1]]) - - # fix values to all U[k,j] and L[k,j] from U_bounds and L_bounds - index = 1 - for k in 0:K - for j in 1:node_count[k+1] - fix(U[k, j], curr_U_bounds[index], force=true) - fix(L[k, j], curr_L_bounds[index], force=true) - index += 1 - end - end - - # input layer (layer 0) node bounds are given beforehand - for input_node in 1:node_count[1] - delete_lower_bound(x[0, input_node]) - @constraint(model, L[0, input_node] <= x[0, input_node]) - @constraint(model, x[0, input_node] <= U[0, input_node]) - end - - # deleting lower bound for output nodes - for output_node in 1:node_count[K+1] - delete_lower_bound(x[K, output_node]) - end - - ### below constraints depending on the layer (every constraint up to the previous layer) - for k_in in 1:k - for node_in in 1:node_count[k_in] - if k_in >= 2 - temp_sum = sum(W[k_in-1][node_in, j] * x[k_in-1-1, j] for j in 1:node_count[k_in-1]) - @constraint(model, x[k_in-1, node_in] <= U[k_in-1, node_in] * z[k_in-1, node_in]) - @constraint(model, s[k_in-1, node_in] <= -L[k_in-1, node_in] * (1 - z[k_in-1, node_in])) - if k_in <= K - 1 - @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in] - s[k_in-1, node_in]) - else # k_in == K - @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in]) - end - end - end - end - - ### below constraints depending on the node - temp_sum = sum(W[k][curr_node, j] * x[k-1, j] for j in 1:node_count[k]) # NOTE! prev layer [k] - if k <= K - 1 - @constraint(model, node_con, temp_sum + b[k][curr_node] == x[k, curr_node] - s[k, curr_node]) - @constraint(model, node_U, x[k, curr_node] <= U[k, curr_node] * z[k, curr_node]) - @constraint(model, node_L, s[k, curr_node] <= -L[k, curr_node] * (1 - z[k, curr_node])) - elseif k == K # == last value of k - @constraint(model, node_con, temp_sum + b[k][curr_node] == x[k, curr_node]) - @constraint(model, node_L, L[k, curr_node] <= x[k, curr_node]) - @constraint(model, node_U, x[k, curr_node] <= U[k, curr_node]) - end - - if obj_function == 1 && k <= K - 1 # Min, hidden layer - @objective(model, Min, x[k, curr_node] - s[k, curr_node]) - elseif obj_function == 2 && k <= K - 1 # Max, hidden layer - @objective(model, Max, x[k, curr_node] - s[k, curr_node]) - elseif obj_function == 1 && k == K # Min, last layer - @objective(model, Min, x[k, curr_node]) - elseif obj_function == 2 && k == K # Max, last layer - @objective(model, Max, x[k, curr_node]) - end - - solve_time = @elapsed optimize!(model) - solve_time = round(solve_time; sigdigits = 3) - @assert termination_status(model) == OPTIMAL || termination_status(model) == TIME_LIMIT - "Problem (layer $k (from 1:$K), node $curr_node, $(obj_function == 1 ? "L" : "U")-bound) is infeasible." - optimal = objective_value(model) - println("Worker: $(myid()), layer $k, node $curr_node, $(obj_function == 1 ? "L" : "U")-bound: solve time $(solve_time)s, optimal value $(optimal)") - - return optimal -end - -""" -bound_tightening_2workers(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false) - -A multi-threaded (using workers) implementation of optimal tightened constraint bounds L and U for for a trained DNN. -This function uses two in-place models at each layer to reduce memory usage. A max of 2 workers in use simultaneously. -Using these bounds with the create_JuMP_model function reduces solution time for optimization problems. - -# Arguments -- `DNN::Chain`: A trained ReLU DNN. -- `init_U_bounds::Vector{Float32}`: Initial upper bounds on the node values of the DNN. -- `init_L_bounds::Vector{Float32}`: Initial lower bounds on the node values of the DNN. -- `verbose::Bool=false`: Controls Gurobi logs. - -# Examples -```julia -L_bounds_workers, U_bounds_workers = bound_tightening_2workers(DNN, init_U_bounds, init_L_bounds, false) -``` -""" - -function bound_tightening_2workers(DNN::Chain, init_U_bounds::Vector{Float32}, init_L_bounds::Vector{Float32}, verbose::Bool=false) - - K = length(DNN) # NOTE! there are K+1 layers in the nn - - # store the DNN weights and biases - DNN_params = params(DNN) - W = [DNN_params[2*i-1] for i in 1:K] - b = [DNN_params[2*i] for i in 1:K] - - # stores the node count of layer k (starting at layer k=0) at index k+1 - input_node_count = length(DNN_params[1][1, :]) - node_count = [if k == 1 input_node_count else length(DNN_params[2*(k-1)]) end for k in 1:K+1] - - # store the current optimal bounds in the algorithm - curr_U_bounds = copy(init_U_bounds) - curr_L_bounds = copy(init_L_bounds) - - # split the available threads into 2 to be assigned to each worker (integer division) - n = Threads.nthreads() - threads_split = [n÷2, n-(n÷2)] - - for k in 1:K - - L_U_bounds = Distributed.pmap(obj_function -> - bt_2workers_inner(K, k, obj_function, W, b, node_count, curr_U_bounds, curr_L_bounds, threads_split[obj_function], verbose), 1:2) - - curr_L_bounds = L_U_bounds[1] - curr_U_bounds = L_U_bounds[2] - - end - - println("Solving optimal constraint bounds complete") - - return curr_U_bounds, curr_L_bounds -end - - -# Inner function to solve_optimal_bounds_2workers: solves L or U bounds for all nodes in a layer using the same JuMP model - -function bt_2workers_inner( - K::Int64, - k::Int64, - obj_function::Int64, - W::Vector{Matrix{Float32}}, - b::Vector{Vector{Float32}}, - node_count::Vector{Int64}, - curr_U_bounds::Vector{Float32}, - curr_L_bounds::Vector{Float32}, - n_threads::Int64, - verbose::Bool - ) - - curr_U_bounds_copy = copy(curr_U_bounds) - curr_L_bounds_copy = copy(curr_L_bounds) - - model = Model(optimizer_with_attributes(Gurobi.Optimizer, "OutputFlag" => (verbose ? 1 : 0), "Threads" => n_threads)) - - # NOTE! below variables and constraints for all opt problems - @variable(model, x[k in 0:K, j in 1:node_count[k+1]] >= 0) - @variable(model, s[k in 1:K-1, j in 1:node_count[k+1]] >= 0) - @variable(model, z[k in 1:K-1, j in 1:node_count[k+1]], Bin) - @variable(model, U[k in 0:K, j in 1:node_count[k+1]]) - @variable(model, L[k in 0:K, j in 1:node_count[k+1]]) - - # fix values to all U[k,j] and L[k,j] from U_bounds and L_bounds - index = 1 - for k in 0:K - for j in 1:node_count[k+1] - fix(U[k, j], curr_U_bounds[index], force=true) - fix(L[k, j], curr_L_bounds[index], force=true) - index += 1 - end - end - - # input layer (layer 0) node bounds are given beforehand - for input_node in 1:node_count[1] - delete_lower_bound(x[0, input_node]) - @constraint(model, L[0, input_node] <= x[0, input_node]) - @constraint(model, x[0, input_node] <= U[0, input_node]) - end - - # deleting lower bound for output nodes - for output_node in 1:node_count[K+1] - delete_lower_bound(x[K, output_node]) - end - - ### below constraints depending on the layer (every constraint up to the previous layer) - for k_in in 1:k - for node_in in 1:node_count[k_in] - if k_in >= 2 - temp_sum = sum(W[k_in-1][node_in, j] * x[k_in-1-1, j] for j in 1:node_count[k_in-1]) - @constraint(model, x[k_in-1, node_in] <= U[k_in-1, node_in] * z[k_in-1, node_in]) - @constraint(model, s[k_in-1, node_in] <= -L[k_in-1, node_in] * (1 - z[k_in-1, node_in])) - if k_in <= K - 1 - @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in] - s[k_in-1, node_in]) - else # k_in == K - @constraint(model, temp_sum + b[k_in-1][node_in] == x[k_in-1, node_in]) - end - end - end - end - - for node in 1:node_count[k+1] - - prev_layers_node_sum = 0 - for prev_layer in 0:k-1 - prev_layers_node_sum += node_count[prev_layer+1] - end - curr_node_index = prev_layers_node_sum + node - - ### below constraints depending on the node - temp_sum = sum(W[k][node, j] * x[k-1, j] for j in 1:node_count[k]) # NOTE! prev layer [k] - if k <= K - 1 - @constraint(model, node_con, temp_sum + b[k][node] == x[k, node] - s[k, node]) - @constraint(model, node_U, x[k, node] <= U[k, node] * z[k, node]) - @constraint(model, node_L, s[k, node] <= -L[k, node] * (1 - z[k, node])) - elseif k == K # == last value of k - @constraint(model, node_con, temp_sum + b[k][node] == x[k, node]) - @constraint(model, node_L, L[k, node] <= x[k, node]) - @constraint(model, node_U, x[k, node] <= U[k, node]) - end - - if obj_function == 1 && k <= K - 1 # Min, hidden layer - @objective(model, Min, x[k, node] - s[k, node]) - elseif obj_function == 2 && k <= K - 1 # Max, hidden layer - @objective(model, Max, x[k, node] - s[k, node]) - elseif obj_function == 1 && k == K # Min, last layer - @objective(model, Min, x[k, node]) - elseif obj_function == 2 && k == K # Max, last layer - @objective(model, Max, x[k, node]) - end - - solve_time = @elapsed optimize!(model) - solve_time = round(solve_time; sigdigits = 3) - @assert termination_status(model) == OPTIMAL || termination_status(model) == TIME_LIMIT - "Problem (layer $k (from 1:$K), node $node, $(obj_function == 1 ? "L" : "U")-bound) is infeasible." - optimal = objective_value(model) - println("Worker: $(myid()), layer $k, node $node, $(obj_function == 1 ? "L" : "U")-bound: solve time $(solve_time)s, optimal value $(optimal)") - - # fix the model variable L or U corresponding to the current node to be the optimal value - if obj_function == 1 # Min - curr_L_bounds_copy[curr_node_index] = optimal - elseif obj_function == 2 # Max - curr_U_bounds_copy[curr_node_index] = optimal - end - - # deleting and unregistering the constraints assigned to the current node - delete(model, node_con) - delete(model, node_L) - delete(model, node_U) - unregister(model, :node_con) - unregister(model, :node_L) - unregister(model, :node_U) - end - - if obj_function == 1 # Min - return curr_L_bounds_copy - elseif obj_function == 2 # Max - return curr_U_bounds_copy - end - -end diff --git a/src/neural_networks/old/lossless_compression.jl b/src/neural_networks/old/lossless_compression.jl deleted file mode 100644 index e8a8095..0000000 --- a/src/neural_networks/old/lossless_compression.jl +++ /dev/null @@ -1,251 +0,0 @@ -using LinearAlgebra -using Flux: Chain, Dense, relu -using Random -using NPZ - -Random.seed!(1234) - -# helper functions -function forward_pass(X, W1_, b1_, W2_, b2_) - A1 = W1_ * X .+ b1_[:] - R1 = relu.(A1) - product = W2_ * R1 - A2 = product .+ b2_[:] - return A1, A2 -end - -function rank_threshold(M, threshold=1e-5) - # TODO: compare these two methods - return rank(M) - - # F = svd(M) - # S = F.S - # return sum(abs.(S) .> threshold) -end - -# prune neurons that are always inactive -function prune_by_upper_bound(G1, G_bar1, W1, b1, G2, G_bar2, W2, b2) - # TODO: consider using leq and a threshold instead of 0 (epsilon) - to_prune = [i for i = 1:size(W1, 1) if G1[i] < 0] - - W1 = W1[setdiff(1:end, to_prune), :] - b1 = b1[setdiff(1:end, to_prune)] - G1 = G1[setdiff(1:end, to_prune)] - G_bar1 = G_bar1[setdiff(1:end, to_prune)] - - W2 = W2[:, setdiff(1:end, to_prune)] - - return G1, G_bar1, W1, b1, G2, G_bar2, W2, b2, length(to_prune) -end - -# prune neurons that have zero weights -function prune_zero_weights(G1, G_bar1, W1, b1, G2, G_bar2, W2, b2) - # TODO: consider using a threshold instead of 0 (epsilon) - to_prune = [index[1] for index in findall(sum(abs.(W1), dims=2) .< 1e-5)] - - b2 .+= W2[:, to_prune] * b1[to_prune] - - W1 = W1[setdiff(1:end, to_prune), :] - b1 = b1[setdiff(1:end, to_prune)] - G1 = G1[setdiff(1:end, to_prune)] - G_bar1 = G_bar1[setdiff(1:end, to_prune)] - - W2 = W2[:, setdiff(1:end, to_prune)] - - return G1, G_bar1, W1, b1, G2, G_bar2, W2, b2, length(to_prune) -end - -# prune layers that are stable -function prune_stable_layer(W1, b1, W2, b2, S, X) - n_neurons_l_minus_1 = size(X, 1) - n_neurons_l_plus_1 = size(W2, 1) - - W_bar = zeros(Float32, n_neurons_l_minus_1, n_neurons_l_plus_1) - b_bar = zeros(Float32, n_neurons_l_plus_1) - - for i in 1:n_neurons_l_plus_1 - b_bar[i] = b2[i] + dot(W2[i, collect(S)], b1[collect(S)]) - - for j in 1:n_neurons_l_minus_1 - W_bar[j, i] = dot(W1[collect(S), j], W2[i, collect(S)]) - end - end - - return W_bar', b_bar -end - -# prune neurons that are linearly dependent on other neurons -function prune_stabily_active(G1, G_bar1, W1, b1, G2, G_bar2, W2, b2, X, S, i) - W_current = W1[[i; collect(S)], :] - - if rank_threshold(W_current) > length(S) - push!(S, i) - return G1, G_bar1, W1, b1, G2, G_bar2, W2, b2, S - end - - alpha = W1[collect(S), :]' \ W1[i, :] - - for j in axes(W2, 1) - W2[j, collect(S)] .+= alpha .* W2[j, i] - b2[j] += W2[j, i] * (b1[i] - dot(alpha, b1[collect(S)])) - end - - W1 = W1[setdiff(1:end, i), :] - b1 = b1[setdiff(1:end, i)] - # G1 = G1[setdiff(1:end, i)] - # G_bar1 = G_bar1[setdiff(1:end, i)] - W2 = W2[:, setdiff(1:end, i)] - S = Set{Int}([idx >= i ? idx - 1 : idx for idx in S]) - - return G1, G_bar1, W1, b1, G2, G_bar2, W2, b2, S -end - -# prune the network -function prune_neuron(W1, b1, W2, b2, X, G1, G2, G_bar1, G_bar2) - A1, A2 = forward_pass(X, W1, b1, W2, b2) - - S = Set{Int}() - pruned = false - # TODO: start updating the unstable flag when we have a stable layer - unstable = true - - # TODO: check that it works even if all neurons are pruned - G1, G_bar1, W1, b1, G2, G_bar2, W2, b2, n_pruned_by_upper_bound = prune_by_upper_bound(G1, G_bar1, W1, b1, G2, G_bar2, W2, b2) - G1, G_bar1, W1, b1, G2, G_bar2, W2, b2, n_pruned_by_zero_weight = prune_zero_weights(G1, G_bar1, W1, b1, G2, G_bar2, W2, b2) - - n_neurons_initial = size(W1, 1) - - for i in n_neurons_initial:-1:1 - if G_bar1[i] > 1e-5 - s_before = length(S) - - G1, G_bar1, W1, b1, G2, G_bar2, W2, b2, S = prune_stabily_active(G1, G_bar1, W1, b1, G2, G_bar2, W2, b2, X, S, i) - - length(S) == s_before && (pruned = true) - else - unstable = true - end - end - - n_neurons_final = size(W1, 1) - n_pruned_by_linear_dependence = n_neurons_initial - n_neurons_final - - _, A2_pruned = forward_pass(X, W1, b1, W2, b2) # TODO: remove this as it is inaccurate in odd bounded layers - is_close = all(abs.(A2 - A2_pruned) .< 1e-3) - - if !unstable - if length(S) == 0 - # TODO: update this to return the full pruned network - Upsilon = A2_pruned - W2 = zeros(size(W2)) - b2 = fill(Upsilon, size(b2)) - println("Constant output, pruned full layer") - return W1, b1, W2, b2, pruned, is_close, true - end - - W_bar, b_bar = prune_stable_layer(W1, b1, W2, b2, S, X) - - # TODO: update this ridiculous naming - A2_superpruned = W_bar * X .+ b_bar[:] - - return W1, b1, W2, b2, pruned, is_close, is_close && all(abs.(A2 - A2_superpruned) .< 1e-3), n_pruned_by_upper_bound, n_pruned_by_zero_weight, n_pruned_by_linear_dependence - end - - return W1, b1, W2, b2, pruned, is_close, false, n_pruned_by_upper_bound, n_pruned_by_zero_weight, n_pruned_by_linear_dependence -end - -# main function that takes the parameters of consecutive layers and their bounds, and compresses them -function prune_from_model(W1, b1, W2, b2, upper1, upper2, lower1, lower2) - n_neurons = size(W1, 2) - - X = rand(n_neurons) - - G1, G2 = upper1, upper2 - G_bar1, G_bar2 = lower1, lower2 - - W1, b1, W2, b2, pruned, is_close, layer_folded, n_pruned_by_upper_bound, n_pruned_by_zero_weight, n_pruned_by_linear_dependence = prune_neuron(W1, b1, W2, b2, X, G1, G2, G_bar1, G_bar2) - - return W1, b1, W2, b2, n_pruned_by_upper_bound, n_pruned_by_zero_weight, n_pruned_by_linear_dependence -end - -# ! FOR DEBUGGING -function add_linear_dependencies_to_matrix(W1, max_dependencies=2) - n_rows, _ = size(W1) - num_dependencies = rand(1:max_dependencies) - - for _ in 1:num_dependencies - dependent_row = rand(1:n_rows) - num_contributors = rand(1:n_rows-1) - - # get the indices of the rows (all rows except the dependent row), and shuffle them - contributor_rows = shuffle(setdiff(1:n_rows, dependent_row))[1:num_contributors] - - W1[dependent_row, :] .= 0.0 - - for i in 1:num_contributors - W1[dependent_row, :] .+= rand() * 10 * W1[contributor_rows[i], :] - end - end - - return W1 -end - -function create_matrices(n_input = 3, n_neurons_1 = 3, n_neurons_2 = 3, fraction_of_linear_dependencies = 0.5) - W1 = ((rand(n_input, n_neurons_1) .- 0.1) .* 2)' - b1 = (rand(n_neurons_1) .- 0.1) .* 2 - W2 = ((rand(n_neurons_1, n_neurons_2) .- 0.1) .* 2)' - b2 = (rand(n_neurons_2) .- 0.1) .* 2 - - if rand() < fraction_of_linear_dependencies - W1 = add_linear_dependencies_to_matrix(W1) - end - - X = (rand(n_input) .- 0.5) .* 2 - - return W1, b1, W2, b2, X -end - -function create_simple_matrices() - W1 = Float32[ - 1 1 0; - 2 0 -1; - 4 2 -1; - ] - b1 = Float32[5, 4, 6] - W2 = Float32[ - 2 1 0; - 0 1 2; - 1 0 2; - ] - b2 = Float32[0.2, 0.3, 0.1] - - X = Float32[1.0, 1.0, 1.0] - - return W1, b1, W2, b2, X -end - -function run_tests(n_iterations = 10) - pruned_count = 0 - layers_folded = 0 - different_output_count = 0 - - for iter in 1:n_iterations - W1, b1, W2, b2, X = create_matrices(5, 5, 5, 0.5) - # W1, b1, W2, b2, X = create_simple_matrices() # use this for manual debugging - - A1, A2 = forward_pass(X, W1, b1, W2, b2) - # for single outputs this works (G == G_bar) - G1, G2 = A1, A2 - G_bar1, G_bar2 = A1, A2 - - W1, b1, W2, b2, pruned, is_close, layer_folded, _, _, _ = prune_neuron(W1, b1, W2, b2, X, G1, G2, G_bar1, G_bar2) - - pruned && (pruned_count += 1) - layer_folded && (layers_folded += 1) - !is_close && (different_output_count += 1) - end - - println("Number of pruned iterations: $pruned_count") - println("Number of layers folded: $layers_folded") - println("Number of iterations where A_2 was different than A_2_pruned: $different_output_count") -end diff --git a/test/neural_networks/NN_test.jl b/test/neural_networks/NN_test.jl new file mode 100644 index 0000000..a37ce0b --- /dev/null +++ b/test/neural_networks/NN_test.jl @@ -0,0 +1,59 @@ +using Flux +using Random +using Gogeta + +@info "Creating a medium-sized neural network with random weights" +begin + Random.seed!(1234); + + model = Chain( + Dense(2 => 10, relu), + Dense(10 => 50, relu), + Dense(50 => 20, relu), + Dense(20 => 5, relu), + Dense(5 => 1) + ) +end + +init_U = [-0.5, 0.5]; +init_L = [-1.5, -0.5]; + +x1 = (rand(100) * (init_U[1] - init_L[1])) .+ init_L[1]; +x2 = (rand(100) * (init_U[2] - init_L[2])) .+ init_L[2]; +x = transpose(hcat(x1, x2)) .|> Float32; + +solver_params = SolverParams(solver="GLPK", silent=true, threads=0, relax=false, time_limit=0); + +@info "Compressing the neural network with simultaneous bound tightening." +jump_model, compressed_model, removed_neurons, bounds_U, bounds_L = compress(model, init_U, init_L; params=solver_params); + +@info "Testing that the compressed model and the corresponding JuMP model are equal to the original neural network." +@test vec(model(x)) ≈ [forward_pass!(jump_model, x[:, i])[] for i in 1:size(x)[2]] +@test compressed_model(x) ≈ model(x) + +@info "Creating a JuMP model from the neural network with bound tightening but without compression." +nn_jump, U_correct, L_correct = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds="standard"); + +@info "Testing that the created JuMP model is equal to the original neural network." +@test vec(model(x)) ≈ [forward_pass!(nn_jump, x[:, i])[] for i in 1:size(x)[2]] + +@info "Compressing with the precomputed bounds." +compressed, removed = compress(model, init_U, init_L; bounds_U=U_correct, bounds_L=L_correct); + +@info "Testing that this compression result is equal to the original." +@test compressed(x) ≈ model(x) + +@info "Testing that the removed neurons are the same as with simultaneous bound tightening compression." +@test removed ≈ removed_neurons + +@info "Creating a JuMP model of the network with loose bound tightening." +nn_loose, U_loose, L_loose = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds="fast"); + +@info "Testing that the loose JuMP model is equal to the original neural network." +@test vec(model(x)) ≈ [forward_pass!(nn_loose, x[:, i])[] for i in 1:size(x)[2]] + +@info "Compressing with the precomputed loose bounds." +compressed_loose, removed_loose = compress(model, init_U, init_L; bounds_U=U_loose, bounds_L=L_loose); + +@info "Testing that this loose compression result is equal to the original neural network." +@test compressed_loose(x) ≈ model(x) \ No newline at end of file diff --git a/test/neural_networks/CNN_test.jl b/test/neural_networks/old/CNN_test.jl similarity index 100% rename from test/neural_networks/CNN_test.jl rename to test/neural_networks/old/CNN_test.jl diff --git a/test/neural_networks/DNN_bound_tightening_test.jl b/test/neural_networks/old/DNN_bound_tightening_test.jl similarity index 100% rename from test/neural_networks/DNN_bound_tightening_test.jl rename to test/neural_networks/old/DNN_bound_tightening_test.jl diff --git a/test/neural_networks/DNN_test.jl b/test/neural_networks/old/DNN_test.jl similarity index 100% rename from test/neural_networks/DNN_test.jl rename to test/neural_networks/old/DNN_test.jl diff --git a/test/neural_networks/helper_functions.jl b/test/neural_networks/old/helper_functions.jl similarity index 100% rename from test/neural_networks/helper_functions.jl rename to test/neural_networks/old/helper_functions.jl diff --git a/test/runtests.jl b/test/runtests.jl index 2189fb6..d8af508 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -7,6 +7,9 @@ using Test #nclude("nn/DNN_bound_tightening_test.jl") #include("nn/CNN_test.jl") + # tests for neural networks + include("neural_networks/NN_test.jl") + # tests for tree ensembles include("tree_ensembles/TE_test.jl") end diff --git a/test/tree_ensembles/TE_test.jl b/test/tree_ensembles/TE_test.jl index cb50ea8..4213b34 100644 --- a/test/tree_ensembles/TE_test.jl +++ b/test/tree_ensembles/TE_test.jl @@ -32,4 +32,4 @@ minimum = [ -0.0373748243818212, -0.042113434576107486, 0.009143172249250781]; @test solution_lazy ≈ solution_all @test objective_value_all ≈ objective_value_lazy @test objective_value_all ≈ minimum_value -@test reduce(&, [minimum[i] > solution_all[i][1] && minimum[i] < solution_all[i][2] for i in 1:3]) +@test all([minimum[i] > solution_all[i][1] && minimum[i] < solution_all[i][2] for i in 1:3]) From fe4e691723082c512de64c046041bc1214eb10c3 Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Thu, 15 Feb 2024 18:10:31 +0200 Subject: [PATCH 28/32] added options to compression and documentation --- examples/fast-bound-calc.jl | 66 ++++++++++++++++++++ examples/refactor_test.jl | 59 ++++++++++++++++++ src/neural_networks/CNN_JuMP_model.jl | 2 +- src/neural_networks/NN_to_MIP.jl | 69 ++++++++++++++++++--- src/neural_networks/bounds.jl | 72 +--------------------- src/neural_networks/compress.jl | 88 ++++++++++++++++++++++----- test/neural_networks/NN_test.jl | 2 +- test/runtests.jl | 12 ++-- 8 files changed, 268 insertions(+), 102 deletions(-) create mode 100644 examples/fast-bound-calc.jl create mode 100644 examples/refactor_test.jl diff --git a/examples/fast-bound-calc.jl b/examples/fast-bound-calc.jl new file mode 100644 index 0000000..34e84c5 --- /dev/null +++ b/examples/fast-bound-calc.jl @@ -0,0 +1,66 @@ +function calculate_bounds_fast(model::JuMP.Model, layer, neuron, W, b, neurons, layers_removed) + + upper = 1.0e10 + lower = -1.0e10 + + function bounds_callback(cb_data, cb_where::Cint) + + # Only run at integer solutions + if cb_where == GRB_CB_MIPSOL + + objbound = Ref{Cdouble}() + objbest = Ref{Cdouble}() + GRBcbget(cb_data, cb_where, GRB_CB_MIPSOL_OBJBND, objbound) + GRBcbget(cb_data, cb_where, GRB_CB_MIPSOL_OBJBST, objbest) + + if objective_sense(model) == MAX_SENSE + + if objbest[] > 0 + upper = min(objbound[], 1.0e10) + GRBterminate(backend(model)) + end + + if objbound[] <= 0 + upper = max(objbound[], 0.0) + GRBterminate(backend(model)) + end + + elseif objective_sense(model) == MIN_SENSE + + if objbest[] < 0 + lower = max(objbound[], -1.0e10) + GRBterminate(backend(model)) + end + + if objbound[] >= 0 + lower = min(objbound[], 0.0) + GRBterminate(backend(model)) + end + end + end + + end + + @objective(model, Max, b[layer][neuron] + sum(W[layer][neuron, i] * model[:x][layer-1-layers_removed, i] for i in neurons(layer-1-layers_removed))) + + set_attribute(model, "LazyConstraints", 1) + set_attribute(model, Gurobi.CallbackFunction(), bounds_callback) + + optimize!(model) + + set_objective_sense(model, MIN_SENSE) + optimize!(model) + + status = if upper <= 0 + "stabily inactive" + elseif lower >= 0 + "stabily active" + else + "normal" + end + println("Neuron: $neuron, $status, bounds: [$lower, $upper]") + + set_attribute(jump_model, Gurobi.CallbackFunction(), (cb_data, cb_where::Cint)->nothing) + + return upper, lower +end \ No newline at end of file diff --git a/examples/refactor_test.jl b/examples/refactor_test.jl new file mode 100644 index 0000000..55c5c54 --- /dev/null +++ b/examples/refactor_test.jl @@ -0,0 +1,59 @@ +using Flux +using Random +using Gogeta + +@info "Creating a medium-sized neural network with random weights" +begin + Random.seed!(1234); + + model = Chain( + Dense(2 => 10, relu), + Dense(10 => 50, relu), + Dense(50 => 20, relu), + Dense(20 => 5, relu), + Dense(5 => 1) + ) +end + +init_U = [-0.5, 0.5]; +init_L = [-1.5, -0.5]; + +x1 = (rand(100) * (init_U[1] - init_L[1])) .+ init_L[1]; +x2 = (rand(100) * (init_U[2] - init_L[2])) .+ init_L[2]; +x = transpose(hcat(x1, x2)) .|> Float32; + +solver_params = SolverParams(solver="GLPK", silent=true, threads=0, relax=false, time_limit=0); + +@info "Compressing the neural network with simultaneous bound tightening." +jump_model, compressed_model, removed_neurons, bounds_U, bounds_L = compress(model, init_U, init_L; params=solver_params, tighten_bounds="standard"); + +@info "Testing that the compressed model and the corresponding JuMP model are equal to the original neural network." +@test vec(model(x)) ≈ [forward_pass!(jump_model, x[:, i])[] for i in 1:size(x)[2]] +@test compressed_model(x) ≈ model(x) + +@info "Creating a JuMP model from the neural network with bound tightening but without compression." +nn_jump, U_correct, L_correct = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds="standard"); + +@info "Testing that the created JuMP model is equal to the original neural network." +@test vec(model(x)) ≈ [forward_pass!(nn_jump, x[:, i])[] for i in 1:size(x)[2]] + +@info "Compressing with the precomputed bounds." +compressed, removed = compress(model, init_U, init_L; bounds_U=U_correct, bounds_L=L_correct); + +@info "Testing that this compression result is equal to the original." +@test compressed(x) ≈ model(x) + +@info "Testing that the removed neurons are the same as with simultaneous bound tightening compression." +@test removed ≈ removed_neurons + +@info "Creating a JuMP model of the network with loose bound tightening." +nn_loose, U_loose, L_loose = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds="fast"); + +@info "Testing that the loose JuMP model is equal to the original neural network." +@test vec(model(x)) ≈ [forward_pass!(nn_loose, x[:, i])[] for i in 1:size(x)[2]] + +@info "Compressing with the precomputed loose bounds." +compressed_loose, removed_loose = compress(model, init_U, init_L; bounds_U=U_loose, bounds_L=L_loose); + +@info "Testing that this loose compression result is equal to the original neural network." +@test compressed_loose(x) ≈ model(x) \ No newline at end of file diff --git a/src/neural_networks/CNN_JuMP_model.jl b/src/neural_networks/CNN_JuMP_model.jl index c484c31..f215764 100644 --- a/src/neural_networks/CNN_JuMP_model.jl +++ b/src/neural_networks/CNN_JuMP_model.jl @@ -2,7 +2,7 @@ using Flux, JuMP, Gurobi using Flux: params """ -create_CNN_JuMP_model(CNN::Chain, data_shape::Tuple{Int64, Int64, Int64, Int64}, L_bounds::Vector{Array{Float32}}, U_bounds::Vector{Array{Float32}}) + create_CNN_JuMP_model(CNN::Chain, data_shape::Tuple{Int64, Int64, Int64, Int64}, L_bounds::Vector{Array{Float32}}, U_bounds::Vector{Array{Float32}}) Converts a CNN with ReLU activation functions to a 0-1 MILP JuMP model. The ReLU CNN is assumed to be a Flux.Chain. The activation function must be "relu" in all hidden (Conv and Dense) layers and "identity" in the output layer. diff --git a/src/neural_networks/NN_to_MIP.jl b/src/neural_networks/NN_to_MIP.jl index ed0036d..8e9c008 100644 --- a/src/neural_networks/NN_to_MIP.jl +++ b/src/neural_networks/NN_to_MIP.jl @@ -1,7 +1,19 @@ """ - SolverParams() + SolverParams Parameters to be used by the solver. + +# Fields +- `solver`: has to be "Gurobi" or "GLPK" +- `silent`: is the solver log shown +- `threads`: use 0 for solver default +- `relax`: linear relaxation for the MIP +- `time_limit`: time limit for each optimization in the model + +# Examples +```julia +julia> solver_params = SolverParams(solver="Gurobi", silent=true, threads=0, relax=false, time_limit=0); +``` """ @kwdef struct SolverParams solver::String @@ -11,9 +23,39 @@ Parameters to be used by the solver. time_limit::Float64 end +""" + function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}, solver_params::SolverParams; tighten_bounds::String="fast", bounds_U=nothing, bounds_L=nothing, out_ub=nothing, out_lb=nothing) + +Creates a mixed-integer optimization problem from a ReLU-activated neural network. + +Returns a JuMP model containing the MIP formulation as well as the upper and lower activation bounds for each neuron. + +The MIP can be created with initial bounds (optional arguments), or the bounds can be calculated as the model is created in either "fast" or "standard" mode. +If output bounds are to be considered during the tightening, they have to be provided as optional arguments and `tighten_bounds` must be set to "output". + +# Arguments +- `NN_model`: neural network as a `Flux.Chain` +- `init_ub`: upper bounds for the input layer +- `init_lb`: lower bounds for the input layer +- `solver_params`: parameters for the JuMP model solver + +# Optional arguments +- `tighten_bounds`: "fast", "standard" or "output" +- `bounds_U`: upper bounds for the hidden and output layers +- `bounds_L`: lower bounds for the hidden and output layers +- `out_ub`: upper bounds for the output layer +- `out_lb`: lower bounds for the output layer + +# Examples +```julia +julia> nn_jump, U, L = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds="standard"); +``` +""" function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}, solver_params::SolverParams; tighten_bounds::String="fast", bounds_U=nothing, bounds_L=nothing, out_ub=nothing, out_lb=nothing) - precomputed_bounds = (bounds_U !== nothing) && (bounds_L !== nothing) + println("Creating a JuMP model from a Flux.Chain neural network...") + + bounds_precomputed = (bounds_U !== nothing) && (bounds_L !== nothing) @assert tighten_bounds in ("fast", "standard", "output") K = length(NN_model) # number of layers (input layer not included) @@ -28,7 +70,7 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect neurons(layer) = layer == 0 ? [i for i in 1:input_length] : [i for i in 1:neuron_count[layer]] @assert input_length == length(init_ub) == length(init_lb) "Initial bounds arrays must be the same length as the input layer" - if precomputed_bounds @assert length.(bounds_U) == length.([neuron_count[layer] for layer in 1:K]) end + if bounds_precomputed @assert length.(bounds_U) == [neuron_count[layer] for layer in 1:K] end # build model up to second layer jump_model = Model() @@ -41,7 +83,7 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect @constraint(jump_model, [j = 1:input_length], x[0, j] <= init_ub[j]) @constraint(jump_model, [j = 1:input_length], x[0, j] >= init_lb[j]) - if precomputed_bounds == false + if bounds_precomputed == false bounds_U = Vector{Vector}(undef, K) bounds_L = Vector{Vector}(undef, K) end @@ -50,7 +92,7 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect println("\nLAYER $layer") - if precomputed_bounds == false + if bounds_precomputed == false # compute loose bounds if layer == 1 @@ -62,11 +104,11 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect end if tighten_bounds == "standard" - bounds = if nprocs() > 1 - pmap(neuron -> calculate_bounds(copy_model(jump_model, solver_params), layer, neuron, W, b, neurons), neurons(layer)) - else - map(neuron -> calculate_bounds(jump_model, layer, neuron, W, b, neurons), neurons(layer)) - end + bounds = if nprocs() > 1 + pmap(neuron -> calculate_bounds(copy_model(jump_model, solver_params), layer, neuron, W, b, neurons), neurons(layer)) + else + map(neuron -> calculate_bounds(jump_model, layer, neuron, W, b, neurons), neurons(layer)) + end # only change if bound is improved bounds_U[layer] = min.(bounds_U[layer], [bound[1] for bound in bounds]) @@ -119,9 +161,16 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect end end + println("Model creation complete.") + return jump_model, bounds_U, bounds_L end +""" + function forward_pass!(jump_model::JuMP.Model, input) + +Calculates the output of a neural network -representing JuMP model given some input. +""" function forward_pass!(jump_model::JuMP.Model, input) try diff --git a/src/neural_networks/bounds.jl b/src/neural_networks/bounds.jl index e6a6b7d..e2d102a 100644 --- a/src/neural_networks/bounds.jl +++ b/src/neural_networks/bounds.jl @@ -43,77 +43,7 @@ function calculate_bounds(model::JuMP.Model, layer, neuron, W, b, neurons; layer objective_bound(model) end - if upper_bound > 1_000 @warn "Upper bound is very loose: $upper_bound, problem might become infeasible." end - if lower_bound < -1_000 @warn "Lower bound is very loose: $lower_bound, problem might become infeasible." end - - println("Neuron: $neuron [$lower_bound, $upper_bound]") + println("Neuron: $neuron, bounds: [$lower_bound, $upper_bound]") return upper_bound, lower_bound -end - -function calculate_bounds_fast(model::JuMP.Model, layer, neuron, W, b, neurons, layers_removed) - - upper = 1.0e10 - lower = -1.0e10 - - function bounds_callback(cb_data, cb_where::Cint) - - # Only run at integer solutions - if cb_where == GRB_CB_MIPSOL - - objbound = Ref{Cdouble}() - objbest = Ref{Cdouble}() - GRBcbget(cb_data, cb_where, GRB_CB_MIPSOL_OBJBND, objbound) - GRBcbget(cb_data, cb_where, GRB_CB_MIPSOL_OBJBST, objbest) - - if objective_sense(model) == MAX_SENSE - - if objbest[] > 0 - upper = min(objbound[], 1.0e10) - GRBterminate(backend(model)) - end - - if objbound[] <= 0 - upper = max(objbound[], 0.0) - GRBterminate(backend(model)) - end - - elseif objective_sense(model) == MIN_SENSE - - if objbest[] < 0 - lower = max(objbound[], -1.0e10) - GRBterminate(backend(model)) - end - - if objbound[] >= 0 - lower = min(objbound[], 0.0) - GRBterminate(backend(model)) - end - end - end - - end - - @objective(model, Max, b[layer][neuron] + sum(W[layer][neuron, i] * model[:x][layer-1-layers_removed, i] for i in neurons(layer-1-layers_removed))) - - set_attribute(model, "LazyConstraints", 1) - set_attribute(model, Gurobi.CallbackFunction(), bounds_callback) - - optimize!(model) - - set_objective_sense(model, MIN_SENSE) - optimize!(model) - - status = if upper <= 0 - "stabily inactive" - elseif lower >= 0 - "stabily active" - else - "normal" - end - println("Neuron: $neuron, $status, bounds: [$lower, $upper]") - - set_attribute(jump_model, Gurobi.CallbackFunction(), (cb_data, cb_where::Cint)->nothing) - - return upper, lower end \ No newline at end of file diff --git a/src/neural_networks/compress.jl b/src/neural_networks/compress.jl index 5d0fdf5..9d1807a 100644 --- a/src/neural_networks/compress.jl +++ b/src/neural_networks/compress.jl @@ -1,7 +1,38 @@ -function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}; params=nothing, bounds_U=nothing, bounds_L=nothing) +""" + function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}, solver_params::SolverParams; tighten_bounds::String="fast", bounds_U=nothing, bounds_L=nothing, out_ub=nothing, out_lb=nothing) - with_tightening = (bounds_U === nothing || bounds_L === nothing) ? true : false +Creates a mixed-integer optimization problem from a ReLU-activated neural network. + +Returns a JuMP model containing the MIP formulation as well as the upper and lower activation bounds for each neuron. + +The MIP can be created with initial bounds (optional arguments), or the bounds can be calculated as the model is created in either "fast" or "standard" mode. +If output bounds are to be considered during the tightening, they have to be provided as optional arguments and `tighten_bounds` must be set to "output". + +# Arguments +- `NN_model`: neural network as a `Flux.Chain` +- `init_ub`: upper bounds for the input layer +- `init_lb`: lower bounds for the input layer +- `solver_params`: parameters for the JuMP model solver + +# Optional arguments +- `tighten_bounds`: "fast", "standard" or "output" +- `bounds_U`: upper bounds for the hidden and output layers +- `bounds_L`: lower bounds for the hidden and output layers +- `out_ub`: upper bounds for the output layer +- `out_lb`: lower bounds for the output layer + +# Examples +```julia +julia> nn_jump, U, L = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds="standard"); +``` +""" +function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}; params=nothing, bounds_U=nothing, bounds_L=nothing, tighten_bounds="fast") + + println("Starting neural network compression...") + + with_tightening = (bounds_U === nothing || bounds_L === nothing) with_tightening && @assert params !== nothing "Solver parameters must be provided." + @assert tighten_bounds in ("fast", "standard") K = length(model) @@ -18,8 +49,13 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F neuron_count = [length(b[k]) for k in eachindex(b)] neurons(layer) = layer == 0 ? [i for i in 1:input_length] : [i for i in setdiff(1:neuron_count[layer], removed_neurons[layer])] - # build JuMP model if with_tightening + bounds_U = Vector{Vector}(undef, K) + bounds_L = Vector{Vector}(undef, K) + end + + # build JuMP model + if tighten_bounds == "standard" jump_model = Model() set_solver_params!(jump_model, params) @@ -29,9 +65,6 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F @constraint(jump_model, [j = 1:input_length], x[0, j] <= init_ub[j]) @constraint(jump_model, [j = 1:input_length], x[0, j] >= init_lb[j]) - - bounds_U = Vector{Vector}(undef, K) - bounds_L = Vector{Vector}(undef, K) end layers_removed = 0 # how many strictly preceding layers have been removed at current loop iteration @@ -41,12 +74,26 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F println("\nLAYER $layer") if with_tightening - bounds = if nprocs() > 1 - pmap(neuron -> calculate_bounds(copy_model(jump_model, solver_params), layer, neuron, W, b, neurons; layers_removed), neurons(layer)) - else - map(neuron -> calculate_bounds(jump_model, layer, neuron, W, b, neurons; layers_removed), neurons(layer)) - end - bounds_U[layer], bounds_L[layer] = [bound[1] for bound in bounds], [bound[2] for bound in bounds] + + # compute loose bounds + if layer - layers_removed == 1 + bounds_U[layer] = [sum(max(W[layer][neuron, previous] * init_ub[previous], W[layer][neuron, previous] * init_lb[previous]) for previous in neurons(layer-1-layers_removed)) + b[layer][neuron] for neuron in neurons(layer)] + bounds_L[layer] = [sum(min(W[layer][neuron, previous] * init_ub[previous], W[layer][neuron, previous] * init_lb[previous]) for previous in neurons(layer-1-layers_removed)) + b[layer][neuron] for neuron in neurons(layer)] + else + bounds_U[layer] = [sum(max(W[layer][neuron, previous] * max(0, bounds_U[layer-1-layers_removed][previous]), W[layer][neuron, previous] * max(0, bounds_L[layer-1-layers_removed][previous])) for previous in neurons(layer-1-layers_removed)) + b[layer][neuron] for neuron in neurons(layer)] + bounds_L[layer] = [sum(min(W[layer][neuron, previous] * max(0, bounds_U[layer-1-layers_removed][previous]), W[layer][neuron, previous] * max(0, bounds_L[layer-1-layers_removed][previous])) for previous in neurons(layer-1-layers_removed)) + b[layer][neuron] for neuron in neurons(layer)] + end + + if tighten_bounds == "standard" + bounds = if nprocs() > 1 + pmap(neuron -> calculate_bounds(copy_model(jump_model, solver_params), layer, neuron, W, b, neurons; layers_removed), neurons(layer)) + else + map(neuron -> calculate_bounds(jump_model, layer, neuron, W, b, neurons; layers_removed), neurons(layer)) + end + # only change if bound is improved + bounds_U[layer] = min.(bounds_U[layer], [bound[1] for bound in bounds]) + bounds_L[layer] = max.(bounds_L[layer], [bound[2] for bound in bounds]) + end end if layer == K @@ -126,7 +173,7 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F println("Removed $(length(removed_neurons[layer]))/$(neuron_count[layer]) neurons") - if with_tightening + if tighten_bounds == "standard" for neuron in neurons(layer) @constraint(jump_model, x[layer, neuron] >= 0) @constraint(jump_model, s[layer, neuron] >= 0) @@ -146,7 +193,7 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F end # output layer - with_tightening && @constraint(jump_model, [neuron in neurons(K)], x[K, neuron] == b[K][neuron] + sum(W[K][neuron, i] * x[K-1-layers_removed, i] for i in neurons(K-1-layers_removed))) + tighten_bounds == "standard" && @constraint(jump_model, [neuron in neurons(K)], x[K, neuron] == b[K][neuron] + sum(W[K][neuron, i] * x[K-1-layers_removed, i] for i in neurons(K-1-layers_removed))) # build compressed model new_layers = []; @@ -163,10 +210,21 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F end end + println("Compression complete.") + new_model = Flux.Chain(new_layers...) if with_tightening - return jump_model, new_model, removed_neurons, bounds_U, bounds_L + + U_compressed = [bounds_U[layer][neurons(layer)] for layer in 1:K] + filter!(neurons -> length(neurons) != 0, U_compressed) + + L_compressed = [bounds_L[layer][neurons(layer)] for layer in 1:K] + filter!(neurons -> length(neurons) != 0, L_compressed) + + jump_model = NN_to_MIP(new_model, init_ub, init_lb, params; bounds_U=U_compressed, bounds_L=L_compressed)[1] + + return jump_model, new_model, removed_neurons, U_compressed, L_compressed else return new_model, removed_neurons end diff --git a/test/neural_networks/NN_test.jl b/test/neural_networks/NN_test.jl index a37ce0b..55c5c54 100644 --- a/test/neural_networks/NN_test.jl +++ b/test/neural_networks/NN_test.jl @@ -25,7 +25,7 @@ x = transpose(hcat(x1, x2)) .|> Float32; solver_params = SolverParams(solver="GLPK", silent=true, threads=0, relax=false, time_limit=0); @info "Compressing the neural network with simultaneous bound tightening." -jump_model, compressed_model, removed_neurons, bounds_U, bounds_L = compress(model, init_U, init_L; params=solver_params); +jump_model, compressed_model, removed_neurons, bounds_U, bounds_L = compress(model, init_U, init_L; params=solver_params, tighten_bounds="standard"); @info "Testing that the compressed model and the corresponding JuMP model are equal to the original neural network." @test vec(model(x)) ≈ [forward_pass!(jump_model, x[:, i])[] for i in 1:size(x)[2]] diff --git a/test/runtests.jl b/test/runtests.jl index d8af508..7ce7ea2 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,14 +2,18 @@ using Gogeta using Test @testset "Gogeta.jl" begin - # tests for neural networks - #include("nn/DNN_test.jl") - #nclude("nn/DNN_bound_tightening_test.jl") - #include("nn/CNN_test.jl") + + println("\n\n####################") + println("Neural network tests") + println("####################\n\n") # tests for neural networks include("neural_networks/NN_test.jl") + println("\n\n###################") + println("Tree ensemble tests") + println("###################\n\n") + # tests for tree ensembles include("tree_ensembles/TE_test.jl") end From c9a69025a039d548a1ad2fb7c6167b604535ce86 Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Thu, 15 Feb 2024 19:10:28 +0200 Subject: [PATCH 29/32] fixed no-r bound tightening and added test for it --- examples/no-r-test.jl | 50 ++++++++++++++++++++++++++++++++ src/neural_networks/NN_to_MIP.jl | 16 ++++++++-- src/neural_networks/compress.jl | 19 ++++++------ test/neural_networks/NN_test.jl | 15 +++++++++- 4 files changed, 86 insertions(+), 14 deletions(-) create mode 100644 examples/no-r-test.jl diff --git a/examples/no-r-test.jl b/examples/no-r-test.jl new file mode 100644 index 0000000..af2a1cb --- /dev/null +++ b/examples/no-r-test.jl @@ -0,0 +1,50 @@ +using Flux +using Random +using Gogeta +using Plots + +begin + Random.seed!(1234); + + model = Chain( + Dense(2 => 10, relu), + Dense(10 => 50, relu), + Dense(50 => 20, relu), + Dense(20 => 5, relu), + Dense(5 => 1) + ) +end + +init_U = [-0.5, 0.5]; +init_L = [-1.5, -0.5]; + +x1 = (rand(100) * (init_U[1] - init_L[1])) .+ init_L[1]; +x2 = (rand(100) * (init_U[2] - init_L[2])) .+ init_L[2]; +x = transpose(hcat(x1, x2)) .|> Float32; + +solver_params = SolverParams(solver="Gurobi", silent=true, threads=0, relax=false, time_limit=0); + +@time jump_nor, U_nor, L_nor = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds="output", out_ub=[-0.2], out_lb=[-0.4]); +@time jump_standard, U_standard, L_standard = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds="standard"); + +map(input -> begin + test = forward_pass!(jump_nor, input) + if test[] !== nothing + test[] ≈ model(input)[] + else + true + end + end, eachcol(x)) + +# Plot the differences +plot(collect(Iterators.flatten(U_standard))) +plot!(collect(Iterators.flatten(U_nor))) + +plot!(collect(Iterators.flatten(L_standard))) +plot!(collect(Iterators.flatten(L_nor))) + +x_range = LinRange{Float32}(init_L[1], init_U[1], 100); +y_range = LinRange{Float32}(init_L[2], init_U[2], 100); + +contourf(x_range, y_range, (x, y) -> model(hcat(x, y)')[], c=cgrad(:viridis), lw=0) +contourf(x_range, y_range, (x, y) -> forward_pass!(jump_nor, [x, y])[] === nothing ? 0.0 : forward_pass!(jump_nor, [x, y])[], c=cgrad(:viridis), lw=0) \ No newline at end of file diff --git a/src/neural_networks/NN_to_MIP.jl b/src/neural_networks/NN_to_MIP.jl index 8e9c008..bf44ac7 100644 --- a/src/neural_networks/NN_to_MIP.jl +++ b/src/neural_networks/NN_to_MIP.jl @@ -141,10 +141,15 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect if tighten_bounds == "output" @assert length(out_lb) == length(out_ub) == neuron_count[K] "Incorrect length of output bounds array." + println("Starting bound tightening based on output bounds as well as input bounds.") + @constraint(jump_model, [neuron in 1:neuron_count[K]], x[K, neuron] >= out_lb[neuron]) @constraint(jump_model, [neuron in 1:neuron_count[K]], x[K, neuron] <= out_ub[neuron]) - for layer in 1:K-1, neuron in neuron_count[layer] + for layer in 1:K-1 + + println("\nLAYER $layer") + bounds = if nprocs() > 1 pmap(neuron -> calculate_bounds(copy_model(jump_model, solver_params), layer, neuron, W, b, neurons), neurons(layer)) else @@ -155,10 +160,15 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect bounds_U[layer] = min.(bounds_U[layer], [bound[1] for bound in bounds]) bounds_L[layer] = max.(bounds_L[layer], [bound[2] for bound in bounds]) - @constraint(jump_model, x[layer, neuron] <= max(0, bounds_U[layer][neuron]) * (1 - z[layer, neuron])) - @constraint(jump_model, s[layer, neuron] <= max(0, -bounds_L[layer][neuron]) * z[layer, neuron]) + for neuron in neuron_count[layer] + @constraint(jump_model, x[layer, neuron] <= max(0, bounds_U[layer][neuron]) * (1 - z[layer, neuron])) + @constraint(jump_model, s[layer, neuron] <= max(0, -bounds_L[layer][neuron]) * z[layer, neuron]) + end end + + bounds_U[K] = out_ub + bounds_L[K] = out_lb end println("Model creation complete.") diff --git a/src/neural_networks/compress.jl b/src/neural_networks/compress.jl index 9d1807a..4e7f7ee 100644 --- a/src/neural_networks/compress.jl +++ b/src/neural_networks/compress.jl @@ -1,29 +1,28 @@ """ - function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}, solver_params::SolverParams; tighten_bounds::String="fast", bounds_U=nothing, bounds_L=nothing, out_ub=nothing, out_lb=nothing) + function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}; params=nothing, bounds_U=nothing, bounds_L=nothing, tighten_bounds="fast") -Creates a mixed-integer optimization problem from a ReLU-activated neural network. +Creates a new neural network model by identifying stabily active and inactive neurons and removing them. -Returns a JuMP model containing the MIP formulation as well as the upper and lower activation bounds for each neuron. +Can be called with precomputed bounds. +Returns the compressed neural network as a `Flux.Chain` and the indices of the removed neurons in this case. -The MIP can be created with initial bounds (optional arguments), or the bounds can be calculated as the model is created in either "fast" or "standard" mode. -If output bounds are to be considered during the tightening, they have to be provided as optional arguments and `tighten_bounds` must be set to "output". +Can also be called without the bounds to invoke bound tightening ("standard" or "fast" mode). In this case solver parameters have to be provided. +Return the resulting JuMP model, the compressed neural network, the removed neurons and the computed bounds. # Arguments - `NN_model`: neural network as a `Flux.Chain` - `init_ub`: upper bounds for the input layer - `init_lb`: lower bounds for the input layer -- `solver_params`: parameters for the JuMP model solver # Optional arguments -- `tighten_bounds`: "fast", "standard" or "output" +- `params`: parameters for the JuMP model solver +- `tighten_bounds`: "fast" or "standard" - `bounds_U`: upper bounds for the hidden and output layers - `bounds_L`: lower bounds for the hidden and output layers -- `out_ub`: upper bounds for the output layer -- `out_lb`: lower bounds for the output layer # Examples ```julia -julia> nn_jump, U, L = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds="standard"); +julia> jump_model, compressed_model, removed_neurons, bounds_U, bounds_L = compress(model, init_U, init_L; params=solver_params, tighten_bounds="standard"); ``` """ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{Float64}; params=nothing, bounds_U=nothing, bounds_L=nothing, tighten_bounds="fast") diff --git a/test/neural_networks/NN_test.jl b/test/neural_networks/NN_test.jl index 55c5c54..5c78d9e 100644 --- a/test/neural_networks/NN_test.jl +++ b/test/neural_networks/NN_test.jl @@ -34,6 +34,19 @@ jump_model, compressed_model, removed_neurons, bounds_U, bounds_L = compress(mod @info "Creating a JuMP model from the neural network with bound tightening but without compression." nn_jump, U_correct, L_correct = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds="standard"); +@info "Creating bound tightened JuMP model with output bounds present." +@time jump_nor, U_nor, L_nor = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds="output", out_ub=[-0.2], out_lb=[-0.4]); + +@info "Testing that the output tightened model is the same in the areas it's defined in." +@test all(map(input -> begin + test = forward_pass!(jump_nor, input) + if test[] !== nothing + test[] ≈ model(input)[] + else + true + end +end, eachcol(x))) + @info "Testing that the created JuMP model is equal to the original neural network." @test vec(model(x)) ≈ [forward_pass!(nn_jump, x[:, i])[] for i in 1:size(x)[2]] @@ -56,4 +69,4 @@ nn_loose, U_loose, L_loose = NN_to_MIP(model, init_U, init_L, solver_params; tig compressed_loose, removed_loose = compress(model, init_U, init_L; bounds_U=U_loose, bounds_L=L_loose); @info "Testing that this loose compression result is equal to the original neural network." -@test compressed_loose(x) ≈ model(x) \ No newline at end of file +@test compressed_loose(x) ≈ model(x) From 245de0a69740400cb847caa52b05a9339eba62eb Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Fri, 16 Feb 2024 10:13:34 +0200 Subject: [PATCH 30/32] tested no-r tightening --- examples/images/fast-bt-bounds.svg | 51 ++++++++++++++++ examples/images/model-contour.svg | 95 ++++++++++++++++++++++++++++++ examples/images/no-r-countour.svg | 95 ++++++++++++++++++++++++++++++ examples/images/no-r.svg | 51 ++++++++++++++++ examples/no-r-bt.md | 35 +++++++++++ examples/no-r-test.jl | 14 ++--- src/neural_networks/compress.jl | 2 +- 7 files changed, 332 insertions(+), 11 deletions(-) create mode 100644 examples/images/fast-bt-bounds.svg create mode 100644 examples/images/model-contour.svg create mode 100644 examples/images/no-r-countour.svg create mode 100644 examples/images/no-r.svg create mode 100644 examples/no-r-bt.md diff --git a/examples/images/fast-bt-bounds.svg b/examples/images/fast-bt-bounds.svg new file mode 100644 index 0000000..5491075 --- /dev/null +++ b/examples/images/fast-bt-bounds.svg @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/examples/images/model-contour.svg b/examples/images/model-contour.svg new file mode 100644 index 0000000..56fcf48 --- /dev/null +++ b/examples/images/model-contour.svg @@ -0,0 +1,95 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/examples/images/no-r-countour.svg b/examples/images/no-r-countour.svg new file mode 100644 index 0000000..76f57e5 --- /dev/null +++ b/examples/images/no-r-countour.svg @@ -0,0 +1,95 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/examples/images/no-r.svg b/examples/images/no-r.svg new file mode 100644 index 0000000..be49224 --- /dev/null +++ b/examples/images/no-r.svg @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/examples/no-r-bt.md b/examples/no-r-bt.md new file mode 100644 index 0000000..fe4d6e1 --- /dev/null +++ b/examples/no-r-bt.md @@ -0,0 +1,35 @@ +# Output bound considering tightening procedure + +## Performance + +### Output bounds considered +```julia +out_ub=[-0.2], out_lb=[-0.4] +``` + +`17.987260 seconds (407.83 k allocations: 14.170 MiB, 0.07% gc time)` + +### Standard bound tightening +`1.199113 seconds (126.89 k allocations: 5.509 MiB)` + +### Fast bound tightening (only bounds from the previous layer considered) + +`0.003375 seconds (57.35 k allocations: 2.375 MiB)` + +## Bounds +![](images/no-r.svg) + +Bounds with output considered are slightly tighter than the normal bounds. + +![](images/fast-bt-bounds.svg) + +Fast bounds are considerably looser than the normal bounds, especially in deeper layers. + +## JuMP model output +![](images/model-contour.svg) + +Neural network output in the input domain. + +![](images/no-r-countour.svg) + +JuMP model output in the input domain. Note the missing regions that would have values outside of the output bounds. \ No newline at end of file diff --git a/examples/no-r-test.jl b/examples/no-r-test.jl index af2a1cb..bbbd6b9 100644 --- a/examples/no-r-test.jl +++ b/examples/no-r-test.jl @@ -26,25 +26,19 @@ solver_params = SolverParams(solver="Gurobi", silent=true, threads=0, relax=fals @time jump_nor, U_nor, L_nor = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds="output", out_ub=[-0.2], out_lb=[-0.4]); @time jump_standard, U_standard, L_standard = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds="standard"); - -map(input -> begin - test = forward_pass!(jump_nor, input) - if test[] !== nothing - test[] ≈ model(input)[] - else - true - end - end, eachcol(x)) +@time jump_fast, U_fast, L_fast = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds="fast"); # Plot the differences plot(collect(Iterators.flatten(U_standard))) plot!(collect(Iterators.flatten(U_nor))) +plot!(collect(Iterators.flatten(U_fast))) plot!(collect(Iterators.flatten(L_standard))) plot!(collect(Iterators.flatten(L_nor))) +plot!(collect(Iterators.flatten(L_fast))) x_range = LinRange{Float32}(init_L[1], init_U[1], 100); y_range = LinRange{Float32}(init_L[2], init_U[2], 100); contourf(x_range, y_range, (x, y) -> model(hcat(x, y)')[], c=cgrad(:viridis), lw=0) -contourf(x_range, y_range, (x, y) -> forward_pass!(jump_nor, [x, y])[] === nothing ? 0.0 : forward_pass!(jump_nor, [x, y])[], c=cgrad(:viridis), lw=0) \ No newline at end of file +contourf(x_range, y_range, (x, y) -> forward_pass!(jump_nor, [x, y])[] === nothing ? NaN : forward_pass!(jump_nor, [x, y])[], c=cgrad(:viridis), lw=0) \ No newline at end of file diff --git a/src/neural_networks/compress.jl b/src/neural_networks/compress.jl index 4e7f7ee..7136a30 100644 --- a/src/neural_networks/compress.jl +++ b/src/neural_networks/compress.jl @@ -7,7 +7,7 @@ Can be called with precomputed bounds. Returns the compressed neural network as a `Flux.Chain` and the indices of the removed neurons in this case. Can also be called without the bounds to invoke bound tightening ("standard" or "fast" mode). In this case solver parameters have to be provided. -Return the resulting JuMP model, the compressed neural network, the removed neurons and the computed bounds. +Returns the resulting JuMP model, the compressed neural network, the removed neurons and the computed bounds. # Arguments - `NN_model`: neural network as a `Flux.Chain` From adbbcc657d6937fcb172d4c576d43a6330278362 Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Fri, 16 Feb 2024 11:14:06 +0200 Subject: [PATCH 31/32] added old constraint removal to no-r bound tightening --- src/neural_networks/NN_to_MIP.jl | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/neural_networks/NN_to_MIP.jl b/src/neural_networks/NN_to_MIP.jl index bf44ac7..a568e63 100644 --- a/src/neural_networks/NN_to_MIP.jl +++ b/src/neural_networks/NN_to_MIP.jl @@ -87,6 +87,12 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect bounds_U = Vector{Vector}(undef, K) bounds_L = Vector{Vector}(undef, K) end + + ucons = Vector{Vector{ConstraintRef}}(undef, K) + lcons = Vector{Vector{ConstraintRef}}(undef, K) + + [ucons[layer] = ConstraintRef[] for layer in 1:K] + [lcons[layer] = ConstraintRef[] for layer in 1:K] for layer in 1:K # hidden layers and output @@ -126,8 +132,8 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect @constraint(jump_model, s[layer, neuron] >= 0) set_binary(z[layer, neuron]) - @constraint(jump_model, x[layer, neuron] <= max(0, bounds_U[layer][neuron]) * (1 - z[layer, neuron])) - @constraint(jump_model, s[layer, neuron] <= max(0, -bounds_L[layer][neuron]) * z[layer, neuron]) + push!(ucons[layer], @constraint(jump_model, x[layer, neuron] <= max(0, bounds_U[layer][neuron]) * (1 - z[layer, neuron]))) + push!(lcons[layer], @constraint(jump_model, s[layer, neuron] <= max(0, -bounds_L[layer][neuron]) * z[layer, neuron])) @constraint(jump_model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1, i] for i in neurons(layer-1))) @@ -161,6 +167,10 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect bounds_L[layer] = max.(bounds_L[layer], [bound[2] for bound in bounds]) for neuron in neuron_count[layer] + + delete(jump_model, ucons[layer][neuron]) + delete(jump_model, lcons[layer][neuron]) + @constraint(jump_model, x[layer, neuron] <= max(0, bounds_U[layer][neuron]) * (1 - z[layer, neuron])) @constraint(jump_model, s[layer, neuron] <= max(0, -bounds_L[layer][neuron]) * z[layer, neuron]) end From e08d683bab165ad14c66c0d7837e4f02636aea80 Mon Sep 17 00:00:00 2001 From: Eetu Reijonen Date: Fri, 16 Feb 2024 14:43:32 +0200 Subject: [PATCH 32/32] fixed nn tests and modified constraint removal --- examples/large_nn_results.md | 2 +- examples/no-r-test.jl | 2 + src/Gogeta.jl | 2 + src/neural_networks/NN_to_MIP.jl | 12 ++-- src/neural_networks/bounds.jl | 2 +- src/neural_networks/compress.jl | 90 +--------------------------- src/neural_networks/compression.jl | 95 ++++++++++++++++++++++++++++++ test/neural_networks/NN_test.jl | 12 ++-- 8 files changed, 115 insertions(+), 102 deletions(-) create mode 100644 src/neural_networks/compression.jl diff --git a/examples/large_nn_results.md b/examples/large_nn_results.md index 4909742..75ec046 100644 --- a/examples/large_nn_results.md +++ b/examples/large_nn_results.md @@ -73,5 +73,5 @@ In this case, using the LRR bounds is enough to achieve most of the compression - tighter bounds should only be calculated if necessary (e.g. problem cannot be solved quickly enough with loose bounds or the compression has to be maximal) *Grimstad and Andresson:* -- integer problems formulated from small to medium NN models (1-3 hidden layers, width of layers >50) are quite fast to solve even with loose bounds +- integer problems formulated from small to medium NN models (1-3 hidden layers, width of layers <50) are quite fast to solve even with loose bounds - solving the bounds and then solving the problem itself might be slower than just solving the problem with loose bounds \ No newline at end of file diff --git a/examples/no-r-test.jl b/examples/no-r-test.jl index bbbd6b9..4fc5f5e 100644 --- a/examples/no-r-test.jl +++ b/examples/no-r-test.jl @@ -28,6 +28,8 @@ solver_params = SolverParams(solver="Gurobi", silent=true, threads=0, relax=fals @time jump_standard, U_standard, L_standard = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds="standard"); @time jump_fast, U_fast, L_fast = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds="fast"); +@btime [forward_pass!(jump_nor, input)[] for input in eachcol(x)]; + # Plot the differences plot(collect(Iterators.flatten(U_standard))) plot!(collect(Iterators.flatten(U_nor))) diff --git a/src/Gogeta.jl b/src/Gogeta.jl index 551b0bb..4a16d78 100644 --- a/src/Gogeta.jl +++ b/src/Gogeta.jl @@ -19,6 +19,8 @@ export NN_to_MIP, forward_pass!, SolverParams include("neural_networks/bounds.jl") +include("neural_networks/compression.jl") + include("neural_networks/compress.jl") export compress diff --git a/src/neural_networks/NN_to_MIP.jl b/src/neural_networks/NN_to_MIP.jl index a568e63..7589b6e 100644 --- a/src/neural_networks/NN_to_MIP.jl +++ b/src/neural_networks/NN_to_MIP.jl @@ -87,12 +87,12 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect bounds_U = Vector{Vector}(undef, K) bounds_L = Vector{Vector}(undef, K) end - + ucons = Vector{Vector{ConstraintRef}}(undef, K) lcons = Vector{Vector{ConstraintRef}}(undef, K) - [ucons[layer] = ConstraintRef[] for layer in 1:K] - [lcons[layer] = ConstraintRef[] for layer in 1:K] + [ucons[layer] = Vector{ConstraintRef}(undef, neuron_count[layer]) for layer in 1:K] + [lcons[layer] = Vector{ConstraintRef}(undef, neuron_count[layer]) for layer in 1:K] for layer in 1:K # hidden layers and output @@ -132,9 +132,9 @@ function NN_to_MIP(NN_model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vect @constraint(jump_model, s[layer, neuron] >= 0) set_binary(z[layer, neuron]) - push!(ucons[layer], @constraint(jump_model, x[layer, neuron] <= max(0, bounds_U[layer][neuron]) * (1 - z[layer, neuron]))) - push!(lcons[layer], @constraint(jump_model, s[layer, neuron] <= max(0, -bounds_L[layer][neuron]) * z[layer, neuron])) - + ucons[layer][neuron] = @constraint(jump_model, x[layer, neuron] <= max(0, bounds_U[layer][neuron]) * (1 - z[layer, neuron])) + lcons[layer][neuron] = @constraint(jump_model, s[layer, neuron] <= max(0, -bounds_L[layer][neuron]) * z[layer, neuron]) + @constraint(jump_model, x[layer, neuron] - s[layer, neuron] == b[layer][neuron] + sum(W[layer][neuron, i] * x[layer-1, i] for i in neurons(layer-1))) end diff --git a/src/neural_networks/bounds.jl b/src/neural_networks/bounds.jl index e2d102a..3d0e712 100644 --- a/src/neural_networks/bounds.jl +++ b/src/neural_networks/bounds.jl @@ -11,7 +11,7 @@ function set_solver_params!(model, params) params.threads != 0 && set_attribute(model, "Threads", params.threads) elseif params.solver == "GLPK" set_optimizer(model, () -> GLPK.Optimizer()) - params.time_limit != 0 && set_attribute(model, "tm_lim", params.time_limit) + params.time_limit != 0 && set_attribute(model, "tm_lim", params.time_limit * 1_000) else error("Solver has to be \"Gurobi\" or \"GLPK\"") end diff --git a/src/neural_networks/compress.jl b/src/neural_networks/compress.jl index 7136a30..2c06b4d 100644 --- a/src/neural_networks/compress.jl +++ b/src/neural_networks/compress.jl @@ -99,78 +99,7 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F break end - stable_units = Set{Int}() # indices of stable neurons - unstable_units = false - - for neuron in 1:neuron_count[layer] - - if bounds_U[layer][neuron] <= 0 || iszero(W[layer][neuron, :]) # stabily inactive - - if neuron < neuron_count[layer] || length(stable_units) > 0 || unstable_units == true - - if iszero(W[layer][neuron, :]) && b[layer][neuron] > 0 - for neuron_next in 1:neuron_count[layer+1] # adjust biases - b[layer+1][neuron_next] += W[layer+1][neuron_next, neuron] * b[layer][neuron] - end - end - - push!(removed_neurons[layer], neuron) - end - - elseif bounds_L[layer][neuron] >= 0 # stabily active - - if rank(W[layer][collect(union(stable_units, neuron)), :]) > length(stable_units) - push!(stable_units, neuron) - else # neuron is linearly dependent - - S = collect(stable_units) - alpha = transpose(W[layer][S, :]) \ W[layer][neuron, :] - - for neuron_next in 1:neuron_count[layer+1] # adjust weights and biases - W[layer+1][neuron_next, S] .+= W[layer+1][neuron_next, neuron] * alpha - b[layer+1][neuron_next] += W[layer+1][neuron_next, neuron] * (b[layer][neuron] - dot(b[layer][S], alpha)) - end - - push!(removed_neurons[layer], neuron) - end - else - unstable_units = true - end - - end - - if unstable_units == false # all units in the layer are stable - println("Fully stable layer") - - if length(stable_units) > 0 - - W_bar = Matrix{eltype(W[1][1])}(undef, neuron_count[layer+1], neuron_count[layer-1-layers_removed]) - b_bar = Vector{eltype(b[1][1])}(undef, neuron_count[layer+1]) - - S = collect(stable_units) - - for neuron_next in 1:neuron_count[layer+1] - - b_bar[neuron_next] = b[layer+1][neuron_next] + dot(W[layer+1][neuron_next, S], b[layer][S]) - - for neuron_previous in 1:neuron_count[layer-1-layers_removed] - W_bar[neuron_next, neuron_previous] = dot(W[layer+1][neuron_next, S], W[layer][S, neuron_previous]) - end - end - - W[layer+1] = W_bar - b[layer+1] = b_bar - - layers_removed += 1 - push!(removed_neurons[layer], neurons(layer)...) - else - output = model((init_ub + init_lb) ./ 2) - println("WHOLE NETWORK IS CONSTANT WITH OUTPUT: $output") - return output - end - end - - println("Removed $(length(removed_neurons[layer]))/$(neuron_count[layer]) neurons") + layers_removed = prune!(W, b, removed_neurons, layers_removed, neuron_count, layer, bounds_U, bounds_L) if tighten_bounds == "standard" for neuron in neurons(layer) @@ -194,24 +123,9 @@ function compress(model::Flux.Chain, init_ub::Vector{Float64}, init_lb::Vector{F # output layer tighten_bounds == "standard" && @constraint(jump_model, [neuron in neurons(K)], x[K, neuron] == b[K][neuron] + sum(W[K][neuron, i] * x[K-1-layers_removed, i] for i in neurons(K-1-layers_removed))) - # build compressed model - new_layers = []; - layers = findall(neurons -> length(neurons) > 0, [neurons(l) for l in 1:K]) # layers with neurons - for (i, layer) in enumerate(layers) - - W[layer] = W[layer][neurons(layer), neurons(i == 1 ? 0 : layers[i-1])] - b[layer] = b[layer][neurons(layer)] - - if layer != last(layers) - push!(new_layers, Dense(W[layer], b[layer], relu)) - else - push!(new_layers, Dense(W[layer], b[layer])) - end - end - println("Compression complete.") - new_model = Flux.Chain(new_layers...) + new_model = build_model!(W, b, K, neurons) if with_tightening diff --git a/src/neural_networks/compression.jl b/src/neural_networks/compression.jl new file mode 100644 index 0000000..7488423 --- /dev/null +++ b/src/neural_networks/compression.jl @@ -0,0 +1,95 @@ +function prune!(W, b, removed_neurons, layers_removed, neuron_count, layer, bounds_U, bounds_L) + + stable_units = Set{Int}() # indices of stable neurons + unstable_units = false + + for neuron in 1:neuron_count[layer] + + if bounds_U[layer][neuron] <= 0 || iszero(W[layer][neuron, :]) # stabily inactive + + if neuron < neuron_count[layer] || length(stable_units) > 0 || unstable_units == true + + if iszero(W[layer][neuron, :]) && b[layer][neuron] > 0 + for neuron_next in 1:neuron_count[layer+1] # adjust biases + b[layer+1][neuron_next] += W[layer+1][neuron_next, neuron] * b[layer][neuron] + end + end + + push!(removed_neurons[layer], neuron) + end + + elseif bounds_L[layer][neuron] >= 0 # stabily active + + if rank(W[layer][collect(union(stable_units, neuron)), :]) > length(stable_units) + push!(stable_units, neuron) + else # neuron is linearly dependent + + S = collect(stable_units) + alpha = transpose(W[layer][S, :]) \ W[layer][neuron, :] + + for neuron_next in 1:neuron_count[layer+1] # adjust weights and biases + W[layer+1][neuron_next, S] .+= W[layer+1][neuron_next, neuron] * alpha + b[layer+1][neuron_next] += W[layer+1][neuron_next, neuron] * (b[layer][neuron] - dot(b[layer][S], alpha)) + end + + push!(removed_neurons[layer], neuron) + end + else + unstable_units = true + end + + end + + if unstable_units == false # all units in the layer are stable + println("Fully stable layer") + + if length(stable_units) > 0 + + W_bar = Matrix{eltype(W[1][1])}(undef, neuron_count[layer+1], neuron_count[layer-1-layers_removed]) + b_bar = Vector{eltype(b[1][1])}(undef, neuron_count[layer+1]) + + S = collect(stable_units) + + for neuron_next in 1:neuron_count[layer+1] + + b_bar[neuron_next] = b[layer+1][neuron_next] + dot(W[layer+1][neuron_next, S], b[layer][S]) + + for neuron_previous in 1:neuron_count[layer-1-layers_removed] + W_bar[neuron_next, neuron_previous] = dot(W[layer+1][neuron_next, S], W[layer][S, neuron_previous]) + end + end + + W[layer+1] = W_bar + b[layer+1] = b_bar + + layers_removed += 1 + removed_neurons[layer] = 1:neuron_count[layer] + else + output = model((init_ub + init_lb) ./ 2) + error("WHOLE NETWORK IS CONSTANT WITH OUTPUT: $output") + end + end + + println("Removed $(length(removed_neurons[layer]))/$(neuron_count[layer]) neurons") + return layers_removed +end + +function build_model!(W, b, K, neurons) + + new_layers = []; + layers = findall(neurons -> length(neurons) > 0, [neurons(l) for l in 1:K]) # layers with neurons + for (i, layer) in enumerate(layers) + + W[layer] = W[layer][neurons(layer), neurons(i == 1 ? 0 : layers[i-1])] + b[layer] = b[layer][neurons(layer)] + + if layer != last(layers) + push!(new_layers, Dense(W[layer], b[layer], relu)) + else + push!(new_layers, Dense(W[layer], b[layer])) + end + end + + return Flux.Chain(new_layers...) + +end \ No newline at end of file diff --git a/test/neural_networks/NN_test.jl b/test/neural_networks/NN_test.jl index 5c78d9e..4d93a09 100644 --- a/test/neural_networks/NN_test.jl +++ b/test/neural_networks/NN_test.jl @@ -22,20 +22,20 @@ x1 = (rand(100) * (init_U[1] - init_L[1])) .+ init_L[1]; x2 = (rand(100) * (init_U[2] - init_L[2])) .+ init_L[2]; x = transpose(hcat(x1, x2)) .|> Float32; -solver_params = SolverParams(solver="GLPK", silent=true, threads=0, relax=false, time_limit=0); +solver_params = SolverParams(solver="GLPK", silent=true, threads=0, relax=false, time_limit=1.0); @info "Compressing the neural network with simultaneous bound tightening." jump_model, compressed_model, removed_neurons, bounds_U, bounds_L = compress(model, init_U, init_L; params=solver_params, tighten_bounds="standard"); @info "Testing that the compressed model and the corresponding JuMP model are equal to the original neural network." -@test vec(model(x)) ≈ [forward_pass!(jump_model, x[:, i])[] for i in 1:size(x)[2]] +@test vec(model(x)) ≈ [forward_pass!(jump_model, input)[] for input in eachcol(x)] @test compressed_model(x) ≈ model(x) @info "Creating a JuMP model from the neural network with bound tightening but without compression." nn_jump, U_correct, L_correct = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds="standard"); @info "Creating bound tightened JuMP model with output bounds present." -@time jump_nor, U_nor, L_nor = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds="output", out_ub=[-0.2], out_lb=[-0.4]); +jump_nor, U_nor, L_nor = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds="output", out_ub=[-0.2], out_lb=[-0.4]); @info "Testing that the output tightened model is the same in the areas it's defined in." @test all(map(input -> begin @@ -48,7 +48,7 @@ nn_jump, U_correct, L_correct = NN_to_MIP(model, init_U, init_L, solver_params; end, eachcol(x))) @info "Testing that the created JuMP model is equal to the original neural network." -@test vec(model(x)) ≈ [forward_pass!(nn_jump, x[:, i])[] for i in 1:size(x)[2]] +@test vec(model(x)) ≈ [forward_pass!(nn_jump, input)[] for input in eachcol(x)] @info "Compressing with the precomputed bounds." compressed, removed = compress(model, init_U, init_L; bounds_U=U_correct, bounds_L=L_correct); @@ -57,13 +57,13 @@ compressed, removed = compress(model, init_U, init_L; bounds_U=U_correct, bounds @test compressed(x) ≈ model(x) @info "Testing that the removed neurons are the same as with simultaneous bound tightening compression." -@test removed ≈ removed_neurons +@test removed == removed_neurons @info "Creating a JuMP model of the network with loose bound tightening." nn_loose, U_loose, L_loose = NN_to_MIP(model, init_U, init_L, solver_params; tighten_bounds="fast"); @info "Testing that the loose JuMP model is equal to the original neural network." -@test vec(model(x)) ≈ [forward_pass!(nn_loose, x[:, i])[] for i in 1:size(x)[2]] +@test vec(model(x)) ≈ [forward_pass!(nn_loose, input)[] for input in eachcol(x)] @info "Compressing with the precomputed loose bounds." compressed_loose, removed_loose = compress(model, init_U, init_L; bounds_U=U_loose, bounds_L=L_loose);