diff --git a/althea_kernel_interface/src/exit_server_tunnel.rs b/althea_kernel_interface/src/exit_server_tunnel.rs index f93a76deb..e063edaf7 100644 --- a/althea_kernel_interface/src/exit_server_tunnel.rs +++ b/althea_kernel_interface/src/exit_server_tunnel.rs @@ -2,15 +2,15 @@ use super::KernelInterfaceError; use crate::ip_addr::{add_ipv4, add_ipv4_mask, delete_ipv4}; use crate::iptables::add_iptables_rule; use crate::netfilter::{ - delete_forward_rule, delete_postrouting_rule, does_nftables_exist, init_filter_chain, - init_nat_chain, insert_nft_exit_forward_rules, + add_prerouting_chain, delete_forward_rule, delete_postrouting_rule, does_nftables_exist, + init_filter_chain, init_nat_chain, insert_nft_exit_forward_rules, }; use crate::open_tunnel::to_wg_local; use crate::run_command; use crate::setup_wg_if::get_peers; use crate::traffic_control::{create_root_classful_limit, has_limit}; use althea_types::WgKey; -use ipnetwork::IpNetwork; +use ipnetwork::{IpNetwork, Ipv4Network}; use std::collections::HashSet; use std::net::{IpAddr, Ipv4Addr}; use KernelInterfaceError as Error; @@ -368,6 +368,71 @@ pub fn teardown_snat( Ok(()) } +/// Sets up the CGNAT rules for the exit server run on startup +pub fn setup_cgnat( + exit_ip: Ipv4Addr, + mask: u32, + ex_nic: &str, + possible_ips: Vec, + exit_subnet: Ipv4Network, + internal_subnet: Ipv4Network, +) -> Result<(), Error> { + init_filter_chain()?; + let _ = add_ipv4_mask(exit_ip, mask, ex_nic); + add_prerouting_chain()?; + // get the ip range from first and last in possible ips + let ip_range = format!( + "{}-{}", + possible_ips.first().unwrap(), + possible_ips.last().unwrap() + ); + /* + nft add rule ip nat POSTROUTING oifname $EXT_IF ip saddr $EXIT_SUBNET counter snat to $EXT_RANGE + */ + run_command( + "nft", + &[ + "add", + "rule", + "ip", + "nat", + "postrouting", + "oifname", + ex_nic, + "ip", + "saddr", + &format!("{}", internal_subnet), + "counter", + "snat", + "to", + ip_range.as_str(), + ], + )?; + // nft add rule ip nat prerouting ip daddr 10.0.0.0/24 dnat to 10.0.0.2 + run_command( + "nft", + &[ + "add", + "rule", + "ip", + "nat", + "prerouting", + "ip", + "daddr", + &format!("{}", exit_subnet), + "counter", + "dnat", + "to", + &format!("{}", exit_ip), + ], + )?; + // for each ip in the possible ips range, add it to the external interface + for ip in possible_ips { + add_ipv4(ip, ex_nic)?; + } + Ok(()) +} + #[test] fn test_iproute_parsing() { let str = "fbad::/64,feee::/64"; diff --git a/althea_kernel_interface/src/netfilter.rs b/althea_kernel_interface/src/netfilter.rs index 1f05a9149..e004582a4 100644 --- a/althea_kernel_interface/src/netfilter.rs +++ b/althea_kernel_interface/src/netfilter.rs @@ -91,6 +91,33 @@ fn create_nat_table() -> Result<(), KernelInterfaceError> { Ok(()) } +pub fn add_prerouting_chain() -> Result<(), KernelInterfaceError> { + run_command( + "nft", + &[ + "create", + "chain", + "ip", + "nat", + "prerouting", + "{", + "type", + "nat", + "hook", + "prerouting", + "priority", + "100", + ";", + "policy", + "accept", + ";", + "}", + ], + )?; + + Ok(()) +} + fn create_filter_table() -> Result<(), KernelInterfaceError> { // create the table run_command("nft", &["create", "table", "ip", "filter"])?; diff --git a/integration_tests/src/cgnat_exit.rs b/integration_tests/src/cgnat_exit.rs new file mode 100644 index 000000000..f53bf64f1 --- /dev/null +++ b/integration_tests/src/cgnat_exit.rs @@ -0,0 +1,76 @@ +use althea_kernel_interface::run_command; +use ipnetwork::Ipv4Network; +use settings::exit::ExitIpv4RoutingSettings; + +use crate::five_nodes::five_node_config; +use crate::setup_utils::namespaces::*; +use crate::setup_utils::rita::{spawn_exit_root_of_trust, thread_spawner}; +use crate::utils::{ + add_exits_contract_exit_list, deploy_contracts, get_default_settings, populate_routers_eth, + register_all_namespaces_to_exit, test_all_internet_connectivity, test_reach_all, test_routes, +}; +use std::net::Ipv4Addr; +use std::str::{from_utf8, FromStr}; +use std::thread; +use std::time::Duration; + +/// Runs a five node fixed network map test scenario, this does basic network setup and tests reachability to +/// all destinations +pub async fn run_cgnat_exit_test_scenario() { + info!("Starting cgnat exit node test scenario"); + let node_config = five_node_config(); + let namespaces = node_config.0; + let expected_routes = node_config.1; + + info!("Waiting to deploy contracts"); + let db_addr = deploy_contracts().await; + + let (client_settings, mut exit_settings, exit_root_addr) = + get_default_settings(namespaces.clone(), db_addr); + + // using /29 allows us to test that multiple clients can use the same external IP if randomly assigned + exit_settings.exit_network.ipv4_routing = ExitIpv4RoutingSettings::CGNAT { + subnet: Ipv4Network::from_str("10.0.0.0/29").unwrap(), + static_assignments: Vec::new(), + gateway_ipv4: Ipv4Addr::new(10, 0, 0, 1), + external_ipv4: Ipv4Addr::new(10, 0, 0, 2), + broadcast_ipv4: Ipv4Addr::new(10, 0, 0, 255), + }; + + namespaces.validate(); + + let res = setup_ns(namespaces.clone(), "cgnat"); + info!("Namespaces setup: {res:?}"); + + info!("Starting root server!"); + spawn_exit_root_of_trust(db_addr).await; + + let rita_identities = thread_spawner( + namespaces.clone(), + client_settings, + exit_settings.clone(), + db_addr, + ) + .expect("Could not spawn Rita threads"); + info!("Thread Spawner: {res:?}"); + + // Add exits to the contract exit list so clients get the propers exits they can migrate to + add_exits_contract_exit_list(db_addr, exit_settings.exit_network, rita_identities.clone()) + .await; + + info!("About to populate routers with eth"); + populate_routers_eth(rita_identities, exit_root_addr).await; + + test_reach_all(namespaces.clone()); + + test_routes(namespaces.clone(), expected_routes); + + info!("Registering routers to the exit"); + register_all_namespaces_to_exit(namespaces.clone()).await; + + info!("Checking for wg_exit tunnel setup"); + test_all_internet_connectivity(namespaces.clone()); + info!("All clients successfully registered!"); + + info!("cgnat exit node test scenario complete"); +} diff --git a/integration_tests/src/lib.rs b/integration_tests/src/lib.rs index 3c466f21f..f73c657f8 100644 --- a/integration_tests/src/lib.rs +++ b/integration_tests/src/lib.rs @@ -14,6 +14,7 @@ pub mod payments_althea; pub mod payments_eth; pub mod setup_utils; pub mod snat_exit; +pub mod cgnat_exit; pub mod utils; /// The amount of time we wait for a network to stabalize before testing diff --git a/integration_tests/src/setup_utils/namespaces.rs b/integration_tests/src/setup_utils/namespaces.rs index 7dcc44352..3c5cddc21 100644 --- a/integration_tests/src/setup_utils/namespaces.rs +++ b/integration_tests/src/setup_utils/namespaces.rs @@ -245,6 +245,7 @@ pub fn setup_ns(spaces: NamespaceInfo, exit_mode: &str) -> Result<(), KernelInte let veth_exit_to_native = format!("vout-{}-o", name.get_name()); let exit_ip = match exit_mode { "snat" => "10.0.0.2/24".to_string(), + "cgnat" => "10.0.0.2/24".to_string(), _ => format!( "10.0.{}.{}/24", name.id.to_be_bytes()[0], diff --git a/rita_exit/src/database/ipddr_assignment.rs b/rita_exit/src/database/ipddr_assignment.rs index 93d13c724..e06f490b8 100644 --- a/rita_exit/src/database/ipddr_assignment.rs +++ b/rita_exit/src/database/ipddr_assignment.rs @@ -149,6 +149,9 @@ impl ClientListAnIpAssignmentMap { ExitIpv4RoutingSettings::CGNAT { subnet, static_assignments, + gateway_ipv4, + external_ipv4, + broadcast_ipv4, } => { // check static assignmetns first for id in static_assignments { @@ -545,6 +548,9 @@ mod tests { let ipv4_settings = ExitIpv4RoutingSettings::CGNAT { subnet: get_ipv4_external_test_subnet(), static_assignments, + gateway_ipv4: Ipv4Addr::new(172, 168, 1, 1), + external_ipv4: Ipv4Addr::new(172, 168, 1, 2), + broadcast_ipv4: Ipv4Addr::new(172, 168, 1, 255), }; ipv4_settings.validate().unwrap(); let internal_ipv4_settings = ExitInternalIpv4Settings { diff --git a/rita_exit/src/rita_loop/mod.rs b/rita_exit/src/rita_loop/mod.rs index 3f148c103..d2e62dbca 100644 --- a/rita_exit/src/rita_loop/mod.rs +++ b/rita_exit/src/rita_loop/mod.rs @@ -17,7 +17,9 @@ use crate::traffic_watcher::watch_exit_traffic; use crate::{network_endpoints::*, ClientListAnIpAssignmentMap, RitaExitError}; use actix::System as AsyncSystem; use actix_web::{web, App, HttpServer}; -use althea_kernel_interface::exit_server_tunnel::{one_time_exit_setup, setup_nat, setup_snat}; +use althea_kernel_interface::exit_server_tunnel::{ + one_time_exit_setup, setup_cgnat, setup_nat, setup_snat, +}; use althea_kernel_interface::netfilter::masquerade_nat_setup; use althea_kernel_interface::setup_wg_if::create_blank_wg_interface; use althea_kernel_interface::wg_iface_counter::WgUsage; @@ -27,7 +29,7 @@ use althea_types::{Identity, SignedExitServerList, WgKey}; use babel_monitor::{open_babel_stream, parse_routes}; use clarity::Address; use exit_trust_root::client_db::get_all_registered_clients; -use ipnetwork::Ipv6Network; +use ipnetwork::{Ipv4Network, Ipv6Network}; use rita_common::debt_keeper::DebtAction; use rita_common::rita_loop::get_web3_server; use settings::exit::{ExitIpv4RoutingSettings, EXIT_LIST_PORT}; @@ -468,14 +470,50 @@ fn setup_exit_wg_tunnel() { masquerade_nat_setup(&settings::get_rita_exit().network.external_nic.unwrap()).unwrap(); } ExitIpv4RoutingSettings::CGNAT { - subnet: _, - static_assignments: _, + subnet, + external_ipv4, + static_assignments, + gateway_ipv4, + broadcast_ipv4, } => { - //todo + // collect the client external ips of static assignments vec + let mut static_ips: Vec = static_assignments + .iter() + .map(|x| x.client_external_ip) + .collect(); + // todo these should probably just roll into the initial static assignments list... + static_ips.push(external_ipv4); + static_ips.push(gateway_ipv4); + static_ips.push(broadcast_ipv4); + let possible_ips = get_possible_ips(static_ips, subnet); + + // for cgnat mode we must claim the second ip in the subnet as the exit ip + setup_cgnat( + external_ipv4, + subnet.prefix().into(), + &settings::get_rita_exit().network.external_nic.unwrap(), + possible_ips, + subnet, + exit_settings.exit_network.internal_ipv4.internal_subnet, + ) + .unwrap(); } } } +// gets the range of possible ips for a given subnet +pub fn get_possible_ips(static_assignments: Vec, subnet: Ipv4Network) -> Vec { + // if we don't have a static assignment, we need to find an open ip and assign it + let mut possible_ips: Vec = subnet.into_iter().collect(); + possible_ips.remove(0); // we don't want to assign the first ip in the subnet as it's the subnet default .0 + // remove any ips listed in static assignments + for ip in static_assignments { + possible_ips.retain(|&x| x != ip); + } + info!("Possible ip range is {:?} to {:?}", possible_ips.first().unwrap(), possible_ips.last().unwrap()); + possible_ips +} + /// Starts the rita exit endpoints, passing the ip assignments and registered clients lists, these are shared via cross-thread lock /// with the main rita exit loop. pub fn start_rita_exit_endpoints(ip_assignments: Arc>) { diff --git a/settings/src/exit.rs b/settings/src/exit.rs index 4476da4ab..4835db1bb 100644 --- a/settings/src/exit.rs +++ b/settings/src/exit.rs @@ -48,6 +48,9 @@ pub enum ExitIpv4RoutingSettings { CGNAT { subnet: Ipv4Network, static_assignments: Vec, + gateway_ipv4: Ipv4Addr, + external_ipv4: Ipv4Addr, + broadcast_ipv4: Ipv4Addr, }, /// A provided subnet of ipv4 addresses is assigned one by one to clients as they connect. With an optional /// list of static assignments for clients that will always be assigned the same IP. Use this option with caution @@ -73,6 +76,7 @@ impl ExitIpv4RoutingSettings { ExitIpv4RoutingSettings::CGNAT { subnet, static_assignments, + .. } => { for assignment in static_assignments { if !subnet.contains(assignment.client_external_ip) { diff --git a/test_runner/src/main.rs b/test_runner/src/main.rs index bee2f2d4f..3d69b8273 100644 --- a/test_runner/src/main.rs +++ b/test_runner/src/main.rs @@ -1,3 +1,4 @@ +use integration_tests::cgnat_exit::run_cgnat_exit_test_scenario; use integration_tests::contract_test::run_altheadb_contract_test; use integration_tests::debts::run_debts_test; /// Binary crate for actually running the integration tests @@ -52,6 +53,8 @@ async fn main() { run_altheadb_contract_test().await } else if test_type == "SNAT_EXIT" { run_snat_exit_test_scenario().await + } else if test_type == "CGNAT_EXIT" { + run_cgnat_exit_test_scenario().await } else { panic!("Error unknown test type {}!", test_type); }