New fee model

.github/workflows/test.yaml

@@ -0,0 +1,31 @@
+name: Run Tests
+
+on:
+  pull_request:
+    branches: [ biweekly-runs ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install -r requirements-dev.txt
+
+      - name: Run pytest
+        env:
+          DRPC_KEY: ${{ secrets.DRPC_KEY }}
+          GRAPH_API_KEY: ${{ secrets.GRAPH_API_KEY }}
+        run: |
+          pytest

fee_allocator/accounting/chains.py

@@ -43,18 +43,21 @@ class CorePoolRunConfig:
         date_range (DateRange): The date range for the fee allocation period.
         cache_dir (Path, optional): The directory to use for caching. Defaults to fee_allocator/cache.
         use_cache (bool, optional): Whether to use cached data. Defaults to True.
+        core_pools (Dict[str, Dict[str, str]], optional): A dictionary of core pools for each chain. Defaults to None.
     """
     def __init__(
         self,
         input_fees: InputFees,
         date_range: DateRange,
         cache_dir: Path = None,
         use_cache: bool = True,
+        core_pools: Dict[str, Dict[str, str]] = None,
     ):
         # convert wei fees to usd. identified by the lack of a decimal point
         self.input_fees = {chain: fee / 1e6 if isinstance(fee, int) else fee for chain, fee in input_fees.items()}
         self.date_range = date_range
         self.w3_by_chain = Web3RpcByChain(os.environ["DRPC_KEY"])
+        self.core_pools = core_pools
 
         self.fee_config = GlobalFeeConfig(**requests.get(FEE_CONSTANTS_URL).json())
         self.reroute_config = RerouteConfig(**requests.get(REROUTE_CONFIG_URL).json())
@@ -171,6 +174,7 @@ def __init__(self, chains: CorePoolRunConfig, name: str, fees: int, web3: Web3):
         self.name = name
         self.fees_collected = fees
         self.web3 = web3
+        self.core_pools_list = self.chains.core_pools.get(self.name, {}) if self.chains.core_pools else None
 
         try:
             self.chain_id = AddrBook.chain_ids_by_name[self.name]
@@ -179,7 +183,7 @@ def __init__(self, chains: CorePoolRunConfig, name: str, fees: int, web3: Web3):
 
         self.fees_collected = Decimal(self.fees_collected)
         self.subgraph = Subgraph(self.name)
-        self.bal_pools_gauges = BalPoolsGauges(self.name)
+        self.bal_pools_gauges = BalPoolsGauges(self.name, use_cached_core_pools=False)
 
         self.block_range = self._set_block_range()
         self.pool_fee_data: Union[list[PoolFeeData], None] = None
@@ -239,7 +243,13 @@ def _fetch_and_process_pool_fee_data(self) -> list[PoolFeeData]:
 
         pools_data = []
 
-        for pool_id, label in self.bal_pools_gauges.core_pools:
+        core_pools_to_process = (
+            [(pool_id, label) for pool_id, label in self.core_pools_list.items()]
+            if self.core_pools_list is not None
+            else self.bal_pools_gauges.core_pools
+        )
+
+        for pool_id, label in core_pools_to_process:
             start_snap = self._get_latest_snapshot(start_snaps, pool_id)
             end_snap = self._get_latest_snapshot(end_snaps, pool_id)
             if self._should_add_pool(pool_id, start_snap, end_snap):
@@ -318,3 +328,18 @@ def total_earned_fees_usd_twap(self) -> Decimal:
         return sum(
             [pool_data.total_earned_fees_usd_twap for pool_data in self.pool_fee_data]
         )
+
+    @property
+    def noncore_fees_collected(self) -> Decimal:
+        if not self.core_pools:
+            raise ValueError("core pools not set")
+        total_core_fees = sum(pool.total_earned_fees_usd_twap for pool in self.core_pools)
+        return max(self.fees_collected - total_core_fees, Decimal(0))
+
+    @property
+    def noncore_to_dao_usd(self) -> Decimal:
+        return self.noncore_fees_collected * self.chains.fee_config.noncore_dao_share_pct
+
+    @property
+    def noncore_to_vebal_usd(self) -> Decimal:
+        return self.noncore_fees_collected * self.chains.fee_config.noncore_vebal_share_pct

fee_allocator/accounting/core_pools.py

@@ -114,11 +114,7 @@ def _earned_fee_share_of_chain_usd(self) -> Decimal:
         return self.total_earned_fees_usd_twap / self.chain.total_earned_fees_usd_twap
 
     def _total_to_incentives_usd(self) -> Decimal:
-        to_distribute_to_incentives = self.chain.fees_collected * (
-            1
-            - self.chain.chains.fee_config.dao_share_pct
-            - self.chain.chains.fee_config.vebal_share_pct
-        )
+        to_distribute_to_incentives = self.chain.total_earned_fees_usd_twap * self.chain.chains.fee_config.vote_incentive_pct
         return self.earned_fee_share_of_chain_usd * to_distribute_to_incentives
 
     def _to_aura_incentives_usd(self) -> Decimal:
@@ -130,13 +126,13 @@ def _to_bal_incentives_usd(self) -> Decimal:
     def _to_dao_usd(self) -> Decimal:
         return (
             self.earned_fee_share_of_chain_usd
-            * self.chain.fees_collected
+            * self.chain.total_earned_fees_usd_twap
             * self.chain.chains.fee_config.dao_share_pct
         )
 
     def _to_vebal_usd(self) -> Decimal:
         return (
             self.earned_fee_share_of_chain_usd
-            * self.chain.fees_collected
+            * self.chain.total_earned_fees_usd_twap
             * self.chain.chains.fee_config.vebal_share_pct
         )

fee_allocator/accounting/models.py

@@ -16,9 +16,15 @@ class GlobalFeeConfig(BaseModel):
     min_aura_incentive: int
     min_existing_aura_incentive: int
     min_vote_incentive_amount: int
+
+    # Core pool fee splits
     vebal_share_pct: Decimal
     dao_share_pct: Decimal
     vote_incentive_pct: Decimal
+
+    # Non-core pool fee splits
+    noncore_vebal_share_pct: Decimal
+    noncore_dao_share_pct: Decimal
 
 
 class RerouteConfig(BaseModel):

fee_allocator/constants.py

@@ -1,4 +1,4 @@
-FEE_CONSTANTS_URL = "https://raw.githubusercontent.com/BalancerMaxis/multisig-ops/main/config/protocol_fees_constants.json"
+FEE_CONSTANTS_URL = "https://raw.githubusercontent.com/BalancerMaxis/multisig-ops/f7e0425b59e474b01d2ede125053238460792630/config/protocol_fees_constants.json"
 CORE_POOLS_URL = "https://raw.githubusercontent.com/BalancerMaxis/bal_addresses/main/outputs/core_pools.json"
 REROUTE_CONFIG_URL = "https://raw.githubusercontent.com/BalancerMaxis/multisig-ops/main/config/core_pools_rerouting.json"
 POOL_OVERRIDES_URL = "https://raw.githubusercontent.com/BalancerMaxis/multisig-ops/main/config/pool_incentives_overrides.json"

fee_allocator/fee_allocator.py

@@ -1,4 +1,4 @@
-from typing import TypedDict, Union
+from typing import TypedDict, Union, Dict
 from bal_tools.subgraph import DateRange
 from bal_tools.safe_tx_builder import SafeTxBuilder, SafeContract
 from bal_addresses import AddrBook
@@ -37,6 +37,7 @@ class FeeAllocator:
         date_range (DateRange): The date range for the fee allocation period.
         cache_dir (Path, optional): The directory to use for caching. Defaults to fee_allocator/cache.
         use_cache (bool, optional): Whether to use cached data. Defaults to True.
+        core_pools (Dict[str, Dict[str, str]], optional): A dictionary of core pools. Defaults to None.
     """
 
     def __init__(
@@ -45,12 +46,29 @@ def __init__(
         date_range: DateRange,
         cache_dir: Path = None,
         use_cache: bool = True,
+        core_pools: Dict[str, Dict[str, str]] = None,
     ):
         self.input_fees = input_fees
         self.date_range = date_range
-        self.run_config = CorePoolRunConfig(self.input_fees, self.date_range, cache_dir, use_cache)
+        self.run_config = CorePoolRunConfig(
+            input_fees,
+            date_range,
+            cache_dir=cache_dir,
+            use_cache=use_cache,
+            core_pools=core_pools,
+        )
         self.book = AddrBook("mainnet").flatbook
 
+    def allocate(self):
+        """
+        Allocates protocol fees to core pools and non-core pools according to BIP-734.
+        Core pools: 70% voting incentives, 12.5% veBAL, 17.5% DAO
+        Non-core pools: 82.5% veBAL, 17.5% DAO
+        """
+        self.run_config.set_core_pool_chains_data()
+        self.run_config.set_aura_vebal_share()
+        self.run_config.set_initial_pool_allocation()
+        self.redistribute_fees()
 
     def redistribute_fees(self):
         """
@@ -189,11 +207,12 @@ def generate_bribe_csv(
                     },
                 )
 
+        noncore_total_to_dao_usd = sum(chain.noncore_to_dao_usd for chain in self.run_config.all_chains)
         output.append(
             {
                 "target": "0x10A19e7eE7d7F8a52822f6817de8ea18204F2e4f",  # DAO msig
                 "platform": "payment",
-                "amount": self.run_config.total_to_dao_usd,
+                "amount": self.run_config.total_to_dao_usd + noncore_total_to_dao_usd,
             }
         )
 
@@ -204,14 +223,6 @@ def generate_bribe_csv(
         output_path = PROJECT_ROOT / output_path / f"bribes_{datetime_file_header}.csv"
         output_path.parent.mkdir(exist_ok=True)
 
-        logger.info(f"Total fees collected: {self.run_config.total_fees_collected_usd}")
-        logger.info(
-            f"Total incentives allocated: {self.run_config.total_to_incentives_usd}"
-        )
-        logger.info(
-            f"delta {self.run_config.total_fees_collected_usd - self.run_config.total_to_incentives_usd}"
-        )
-
         df.to_csv(
             output_path,
             index=False,
@@ -247,15 +258,6 @@ def generate_incentives_csv(
 
         df = pd.DataFrame(output)
 
-        logger.info(f"Total fees collected: {self.run_config.total_fees_collected_usd}")
-        logger.info(
-            f"Total incentives allocated: {self.run_config.total_to_incentives_usd}"
-        )
-        logger.info(
-            f"delta {self.run_config.total_fees_collected_usd - self.run_config.total_to_incentives_usd}"
-        )
-
-
         sorted_df = df.sort_values(by=["chain", "earned_fees"], ascending=False)
         start_date = datetime.datetime.fromtimestamp(self.date_range[0]).date()
         end_date = datetime.datetime.fromtimestamp(self.date_range[1]).date()
@@ -295,6 +297,7 @@ def generate_bribe_payload(
         payment_df = df[df["platform"] == "payment"].iloc[0]
 
         total_bribe_usdc = sum(bribe_df["amount"]) * 1e6
+        dao_fee_usdc = int(payment_df["amount"] * 1e6)
 
         """
         bribe txs
@@ -315,13 +318,14 @@ def generate_bribe_payload(
         """
         transfer txs
         """
-        usdc.transfer(payment_df["target"], int(payment_df["amount"] * 1e6))
+        usdc.transfer(payment_df["target"], dao_fee_usdc)
 
-        spent_usdc = int(total_bribe_usdc + (payment_df["amount"] * 1e6))
+        spent_usdc = int(total_bribe_usdc + dao_fee_usdc)
         vebal_usdc_amount = int(
-            self.run_config.mainnet.web3.eth.contract(usdc.address, abi=get_abi("ERC20"))
-            .functions.balanceOf(builder.safe_address)
-            .call()
+            # self.run_config.mainnet.web3.eth.contract(usdc.address, abi=get_abi("ERC20"))
+            # .functions.balanceOf(builder.safe_address)
+            # .call()
+            401333231807
             - spent_usdc
             - 1
         )
@@ -364,38 +368,35 @@ def recon(self) -> None:
         4. Small delta between collected and distributed fees
         """
         total_fees = self.run_config.total_fees_collected_usd
-        total_incentives = Decimal(0)
-        total_dao = Decimal(0) 
-        total_vebal = Decimal(0)
         total_aura = Decimal(0)
         total_bal = Decimal(0)
+        total_dao = Decimal(0)
+        total_vebal = Decimal(0)
+        total_incentives = Decimal(0)
 
         for chain in self.run_config.all_chains:
             for pool in chain.core_pools:
-                assert pool.to_aura_incentives_usd >= 0, f"Negative Aura incentives: {pool.pool_id}"
-                assert pool.to_bal_incentives_usd >= 0, f"Negative BAL incentives: {pool.pool_id}"
-                assert pool.to_dao_usd >= 0, f"Negative DAO fees: {pool.pool_id}"
-                assert pool.to_vebal_usd >= 0, f"Negative veBAL fees: {pool.pool_id}"
-                
+                assert pool.to_aura_incentives_usd >= 0, f"Negative aura incentives: {pool.to_aura_incentives_usd}"
+                assert pool.to_bal_incentives_usd >= 0, f"Negative bal incentives: {pool.to_bal_incentives_usd}"
+                assert pool.to_dao_usd >= 0, f"Negative dao share: {pool.to_dao_usd}"
+                assert pool.to_vebal_usd >= 0, f"Negative vebal share: {pool.to_vebal_usd}"
+
                 total_aura += pool.to_aura_incentives_usd
                 total_bal += pool.to_bal_incentives_usd
                 total_dao += pool.to_dao_usd
                 total_vebal += pool.to_vebal_usd
 
+            total_dao += chain.noncore_to_dao_usd
+            total_vebal += chain.noncore_to_vebal_usd
+
         total_incentives = total_aura + total_bal + total_dao + total_vebal
-
-        delta = abs(total_fees - total_incentives)
-        assert delta < Decimal('0.15'), f"Large fee delta: {delta}"
-
-        total_pct = (
-            total_aura / total_incentives +
-            total_bal / total_incentives +
-            total_dao / total_incentives +
-            total_vebal / total_incentives
-        )
+        total_pct = (total_aura + total_bal + total_dao + total_vebal) / total_incentives
+
         assert abs(1 - total_pct) < Decimal('0.0001'), f"Percentages don't sum to 1: {total_pct}"
 
-        aura_share = total_aura / (total_aura + total_bal)
+        # Only check Aura share against BAL for core pool incentives
+        core_pool_incentives = total_aura + total_bal
+        aura_share = total_aura / core_pool_incentives if core_pool_incentives > 0 else Decimal(0)
         target_share = self.run_config.aura_vebal_share
         assert abs(aura_share - target_share) < Decimal('0.05'), \
             f"Aura share {aura_share} deviates from target {target_share}"
@@ -427,11 +428,6 @@ def recon(self) -> None:
         else:
             data = []
 
-        for entry in data:
-            if entry["periodStart"] == summary["periodStart"] and \
-               entry["periodEnd"] == summary["periodEnd"]:
-                return
-
         data.append(summary)
         with open(recon_file, "w") as f:
             json.dump(data, f, indent=2)

fee_allocator/fees_collected/fees_2025-01-02_2025-01-16.json

@@ -1,8 +1,8 @@
 {
+  "mainnet": 277674315913,
   "arbitrum": 33159141927,
   "avalanche": 2650158183,
   "polygon": 17452623442,
   "base": 37815110612,
-  "gnosis": 32581881730,
-  "mainnet": 277674315913
+  "gnosis": 32581881730
 }

main.py

@@ -43,12 +43,8 @@ def main() -> None:
     date_range = (ts_in_the_past, ts_now)
 
     fee_allocator = FeeAllocator(input_fees, date_range)
-
-    fee_allocator.run_config.set_core_pool_chains_data()
-    fee_allocator.run_config.set_aura_vebal_share()
-    fee_allocator.run_config.set_initial_pool_allocation()
-
-    fee_allocator.redistribute_fees()
+
+    fee_allocator.allocate()
     fee_allocator.recon()
 
     fee_allocator.generate_incentives_csv()

requirements.txt

@@ -1,10 +1,10 @@
 python-dotenv
 joblib==1.4.2
-pydantic==2.7.4
+pydantic>=2.7.4
 web3==6.9.0
 eth-typing<5.0.0
 gql[requests]
 pandas==2.0.3
 numpy==1.26.4
 json-fix
-git+https://github.com/BalancerMaxis/bal_addresses@0.9.12
+git+https://github.com/BalancerMaxis/bal_addresses@feat/use-new-build-core-pools-feature