CGNAT Exit and integration test

right now this is manually adding all possible external ips on the exit subnet,
which we're hoping to avoid since that gets clunky real fast.

althea_kernel_interface/src/exit_server_tunnel.rs

@@ -2,15 +2,15 @@ use super::KernelInterfaceError;
 use crate::ip_addr::{add_ipv4, add_ipv4_mask, delete_ipv4};
 use crate::iptables::add_iptables_rule;
 use crate::netfilter::{
-    delete_forward_rule, delete_postrouting_rule, does_nftables_exist, init_filter_chain,
-    init_nat_chain, insert_nft_exit_forward_rules,
+    add_prerouting_chain, delete_forward_rule, delete_postrouting_rule, does_nftables_exist,
+    init_filter_chain, init_nat_chain, insert_nft_exit_forward_rules,
 };
 use crate::open_tunnel::to_wg_local;
 use crate::run_command;
 use crate::setup_wg_if::get_peers;
 use crate::traffic_control::{create_root_classful_limit, has_limit};
 use althea_types::WgKey;
-use ipnetwork::IpNetwork;
+use ipnetwork::{IpNetwork, Ipv4Network};
 use std::collections::HashSet;
 use std::net::{IpAddr, Ipv4Addr};
 use KernelInterfaceError as Error;
@@ -368,6 +368,71 @@ pub fn teardown_snat(
     Ok(())
 }
 
+/// Sets up the CGNAT rules for the exit server run on startup
+pub fn setup_cgnat(
+    exit_ip: Ipv4Addr,
+    mask: u32,
+    ex_nic: &str,
+    possible_ips: Vec<Ipv4Addr>,
+    exit_subnet: Ipv4Network,
+    internal_subnet: Ipv4Network,
+) -> Result<(), Error> {
+    init_filter_chain()?;
+    let _ = add_ipv4_mask(exit_ip, mask, ex_nic);
+    add_prerouting_chain()?;
+    // get the ip range from first and last in possible ips
+    let ip_range = format!(
+        "{}-{}",
+        possible_ips.first().unwrap(),
+        possible_ips.last().unwrap()
+    );
+    /*
+    nft add rule ip nat POSTROUTING oifname $EXT_IF ip saddr $EXIT_SUBNET counter snat to $EXT_RANGE
+    */
+    run_command(
+        "nft",
+        &[
+            "add",
+            "rule",
+            "ip",
+            "nat",
+            "postrouting",
+            "oifname",
+            ex_nic,
+            "ip",
+            "saddr",
+            &format!("{}", internal_subnet),
+            "counter",
+            "snat",
+            "to",
+            ip_range.as_str(),
+        ],
+    )?;
+    // nft add rule ip nat prerouting ip daddr 10.0.0.0/24 dnat to 10.0.0.2
+    run_command(
+        "nft",
+        &[
+            "add",
+            "rule",
+            "ip",
+            "nat",
+            "prerouting",
+            "ip",
+            "daddr",
+            &format!("{}", exit_subnet),
+            "counter",
+            "dnat",
+            "to",
+            &format!("{}", exit_ip),
+        ],
+    )?;
+    // for each ip in the possible ips range, add it to the external interface
+    for ip in possible_ips {
+        add_ipv4(ip, ex_nic)?;
+    }
+    Ok(())
+}
+
 #[test]
 fn test_iproute_parsing() {
     let str = "fbad::/64,feee::/64";

althea_kernel_interface/src/netfilter.rs

@@ -91,6 +91,33 @@ fn create_nat_table() -> Result<(), KernelInterfaceError> {
     Ok(())
 }
 
+pub fn add_prerouting_chain() -> Result<(), KernelInterfaceError> {
+    run_command(
+        "nft",
+        &[
+            "create",
+            "chain",
+            "ip",
+            "nat",
+            "prerouting",
+            "{",
+            "type",
+            "nat",
+            "hook",
+            "prerouting",
+            "priority",
+            "100",
+            ";",
+            "policy",
+            "accept",
+            ";",
+            "}",
+        ],
+    )?;
+
+    Ok(())
+}
+
 fn create_filter_table() -> Result<(), KernelInterfaceError> {
     // create the table
     run_command("nft", &["create", "table", "ip", "filter"])?;

integration_tests/src/cgnat_exit.rs

@@ -0,0 +1,76 @@
+use althea_kernel_interface::run_command;
+use ipnetwork::Ipv4Network;
+use settings::exit::ExitIpv4RoutingSettings;
+
+use crate::five_nodes::five_node_config;
+use crate::setup_utils::namespaces::*;
+use crate::setup_utils::rita::{spawn_exit_root_of_trust, thread_spawner};
+use crate::utils::{
+    add_exits_contract_exit_list, deploy_contracts, get_default_settings, populate_routers_eth,
+    register_all_namespaces_to_exit, test_all_internet_connectivity, test_reach_all, test_routes,
+};
+use std::net::Ipv4Addr;
+use std::str::{from_utf8, FromStr};
+use std::thread;
+use std::time::Duration;
+
+/// Runs a five node fixed network map test scenario, this does basic network setup and tests reachability to
+/// all destinations
+pub async fn run_cgnat_exit_test_scenario() {
+    info!("Starting cgnat exit node test scenario");
+    let node_config = five_node_config();
+    let namespaces = node_config.0;
+    let expected_routes = node_config.1;
+
+    info!("Waiting to deploy contracts");
+    let db_addr = deploy_contracts().await;
+
+    let (client_settings, mut exit_settings, exit_root_addr) =
+        get_default_settings(namespaces.clone(), db_addr);
+
+    // using /29 allows us to test that multiple clients can use the same external IP if randomly assigned
+    exit_settings.exit_network.ipv4_routing = ExitIpv4RoutingSettings::CGNAT {
+        subnet: Ipv4Network::from_str("10.0.0.0/29").unwrap(),
+        static_assignments: Vec::new(),
+        gateway_ipv4: Ipv4Addr::new(10, 0, 0, 1),
+        external_ipv4: Ipv4Addr::new(10, 0, 0, 2),
+        broadcast_ipv4: Ipv4Addr::new(10, 0, 0, 255),
+    };
+
+    namespaces.validate();
+
+    let res = setup_ns(namespaces.clone(), "cgnat");
+    info!("Namespaces setup: {res:?}");
+
+    info!("Starting root server!");
+    spawn_exit_root_of_trust(db_addr).await;
+
+    let rita_identities = thread_spawner(
+        namespaces.clone(),
+        client_settings,
+        exit_settings.clone(),
+        db_addr,
+    )
+    .expect("Could not spawn Rita threads");
+    info!("Thread Spawner: {res:?}");
+
+    // Add exits to the contract exit list so clients get the propers exits they can migrate to
+    add_exits_contract_exit_list(db_addr, exit_settings.exit_network, rita_identities.clone())
+        .await;
+
+    info!("About to populate routers with eth");
+    populate_routers_eth(rita_identities, exit_root_addr).await;
+
+    test_reach_all(namespaces.clone());
+
+    test_routes(namespaces.clone(), expected_routes);
+
+    info!("Registering routers to the exit");
+    register_all_namespaces_to_exit(namespaces.clone()).await;
+
+    info!("Checking for wg_exit tunnel setup");
+    test_all_internet_connectivity(namespaces.clone());
+    info!("All clients successfully registered!");
+
+    info!("cgnat exit node test scenario complete");
+}

integration_tests/src/lib.rs

@@ -14,6 +14,7 @@ pub mod payments_althea;
 pub mod payments_eth;
 pub mod setup_utils;
 pub mod snat_exit;
+pub mod cgnat_exit;
 pub mod utils;
 
 /// The amount of time we wait for a network to stabalize before testing

integration_tests/src/setup_utils/namespaces.rs

@@ -245,6 +245,7 @@ pub fn setup_ns(spaces: NamespaceInfo, exit_mode: &str) -> Result<(), KernelInte
             let veth_exit_to_native = format!("vout-{}-o", name.get_name());
             let exit_ip = match exit_mode {
                 "snat" => "10.0.0.2/24".to_string(),
+                "cgnat" => "10.0.0.2/24".to_string(),
                 _ => format!(
                     "10.0.{}.{}/24",
                     name.id.to_be_bytes()[0],

rita_exit/src/database/ipddr_assignment.rs

@@ -149,6 +149,9 @@ impl ClientListAnIpAssignmentMap {
             ExitIpv4RoutingSettings::CGNAT {
                 subnet,
                 static_assignments,
+                gateway_ipv4,
+                external_ipv4,
+                broadcast_ipv4,
             } => {
                 // check static assignmetns first
                 for id in static_assignments {
@@ -545,6 +548,9 @@ mod tests {
         let ipv4_settings = ExitIpv4RoutingSettings::CGNAT {
             subnet: get_ipv4_external_test_subnet(),
             static_assignments,
+            gateway_ipv4: Ipv4Addr::new(172, 168, 1, 1),
+            external_ipv4: Ipv4Addr::new(172, 168, 1, 2),
+            broadcast_ipv4: Ipv4Addr::new(172, 168, 1, 255),
         };
         ipv4_settings.validate().unwrap();
         let internal_ipv4_settings = ExitInternalIpv4Settings {

rita_exit/src/rita_loop/mod.rs

@@ -17,7 +17,9 @@ use crate::traffic_watcher::watch_exit_traffic;
 use crate::{network_endpoints::*, ClientListAnIpAssignmentMap, RitaExitError};
 use actix::System as AsyncSystem;
 use actix_web::{web, App, HttpServer};
-use althea_kernel_interface::exit_server_tunnel::{one_time_exit_setup, setup_nat, setup_snat};
+use althea_kernel_interface::exit_server_tunnel::{
+    one_time_exit_setup, setup_cgnat, setup_nat, setup_snat,
+};
 use althea_kernel_interface::netfilter::masquerade_nat_setup;
 use althea_kernel_interface::setup_wg_if::create_blank_wg_interface;
 use althea_kernel_interface::wg_iface_counter::WgUsage;
@@ -27,7 +29,7 @@ use althea_types::{Identity, SignedExitServerList, WgKey};
 use babel_monitor::{open_babel_stream, parse_routes};
 use clarity::Address;
 use exit_trust_root::client_db::get_all_registered_clients;
-use ipnetwork::Ipv6Network;
+use ipnetwork::{Ipv4Network, Ipv6Network};
 use rita_common::debt_keeper::DebtAction;
 use rita_common::rita_loop::get_web3_server;
 use settings::exit::{ExitIpv4RoutingSettings, EXIT_LIST_PORT};
@@ -468,14 +470,50 @@ fn setup_exit_wg_tunnel() {
             masquerade_nat_setup(&settings::get_rita_exit().network.external_nic.unwrap()).unwrap();
         }
         ExitIpv4RoutingSettings::CGNAT {
-            subnet: _,
-            static_assignments: _,
+            subnet,
+            external_ipv4,
+            static_assignments,
+            gateway_ipv4,
+            broadcast_ipv4,
         } => {
-            //todo
+            // collect the client external ips of static assignments vec
+            let mut static_ips: Vec<Ipv4Addr> = static_assignments
+                .iter()
+                .map(|x| x.client_external_ip)
+                .collect();
+            // todo these should probably just roll into the initial static assignments list...
+            static_ips.push(external_ipv4);
+            static_ips.push(gateway_ipv4);
+            static_ips.push(broadcast_ipv4);
+            let possible_ips = get_possible_ips(static_ips, subnet);
+
+            // for cgnat mode we must claim the second ip in the subnet as the exit ip
+            setup_cgnat(
+                external_ipv4,
+                subnet.prefix().into(),
+                &settings::get_rita_exit().network.external_nic.unwrap(),
+                possible_ips,
+                subnet,
+                exit_settings.exit_network.internal_ipv4.internal_subnet,
+            )
+            .unwrap();
         }
     }
 }
 
+// gets the range of possible ips for a given subnet
+pub fn get_possible_ips(static_assignments: Vec<Ipv4Addr>, subnet: Ipv4Network) -> Vec<Ipv4Addr> {
+    // if we don't have a static assignment, we need to find an open ip and assign it
+    let mut possible_ips: Vec<Ipv4Addr> = subnet.into_iter().collect();
+    possible_ips.remove(0); // we don't want to assign the first ip in the subnet as it's the subnet default .0
+    // remove any ips listed in static assignments
+    for ip in static_assignments {
+        possible_ips.retain(|&x| x != ip);
+    }
+    info!("Possible ip range is {:?} to {:?}", possible_ips.first().unwrap(), possible_ips.last().unwrap());
+    possible_ips
+}
+
 /// Starts the rita exit endpoints, passing the ip assignments and registered clients lists, these are shared via cross-thread lock
 /// with the main rita exit loop.
 pub fn start_rita_exit_endpoints(ip_assignments: Arc<RwLock<ClientListAnIpAssignmentMap>>) {

settings/src/exit.rs

@@ -48,6 +48,9 @@ pub enum ExitIpv4RoutingSettings {
     CGNAT {
         subnet: Ipv4Network,
         static_assignments: Vec<ClientIpv4StaticAssignment>,
+        gateway_ipv4: Ipv4Addr,
+        external_ipv4: Ipv4Addr,
+        broadcast_ipv4: Ipv4Addr,
     },
     /// A provided subnet of ipv4 addresses is assigned one by one to clients as they connect. With an optional
     /// list of static assignments for clients that will always be assigned the same IP. Use this option with caution
@@ -73,6 +76,7 @@ impl ExitIpv4RoutingSettings {
             ExitIpv4RoutingSettings::CGNAT {
                 subnet,
                 static_assignments,
+                ..
             } => {
                 for assignment in static_assignments {
                     if !subnet.contains(assignment.client_external_ip) {

test_runner/src/main.rs

@@ -1,3 +1,4 @@
+use integration_tests::cgnat_exit::run_cgnat_exit_test_scenario;
 use integration_tests::contract_test::run_altheadb_contract_test;
 use integration_tests::debts::run_debts_test;
 /// Binary crate for actually running the integration tests
@@ -52,6 +53,8 @@ async fn main() {
             run_altheadb_contract_test().await
         } else if test_type == "SNAT_EXIT" {
             run_snat_exit_test_scenario().await
+        } else if test_type == "CGNAT_EXIT" {
+            run_cgnat_exit_test_scenario().await
         } else {
             panic!("Error unknown test type {}!", test_type);
         }