Merge pull request #8 from gnosis/evan/monitor

Add monitoring of deployed agents

examples/cloud_deployment/gcp/deploy.py

@@ -13,9 +13,9 @@
 if __name__ == "__main__":
     current_dir = os.path.dirname(os.path.realpath(__file__))
     fname = deploy_to_gcp(
-        requirements_file=f"{current_dir}/../../pyproject.toml",
+        requirements_file=f"{current_dir}/../../../pyproject.toml",
         extra_deps=[
-            "git+https://github.com/gnosis/prediction-market-agent.git@evan/deploy-agent"
+            "git+https://github.com/gnosis/prediction-market-agent-tooling.git"
         ],
         function_file=f"{current_dir}/agent.py",
         market_type=MarketType.MANIFOLD,
@@ -30,8 +30,8 @@
     response = run_deployed_gcp_function(fname)
     assert response.ok
 
-    # Schedule the function
-    schedule_deployed_gcp_function(fname, cron_schedule="* * * * *")
+    # Schedule the function to run once every 2 hours
+    schedule_deployed_gcp_function(fname, cron_schedule="0 */2 * * *")
 
     # Delete the function
     remove_deployed_gcp_function(fname)

examples/monitor/monitor.py

@@ -0,0 +1,20 @@
+from datetime import datetime, timedelta
+from zoneinfo import ZoneInfo
+import streamlit as st
+
+from prediction_market_agent_tooling.markets.manifold import get_authenticated_user
+from prediction_market_agent_tooling.monitor.markets.manifold import (
+    DeployedManifoldAgent,
+)
+from prediction_market_agent_tooling.monitor.monitor import monitor_agent
+
+if __name__ == "__main__":
+    start_time = datetime.now() - timedelta(weeks=1)
+    agent = DeployedManifoldAgent(
+        name="foo",
+        start_time=start_time.astimezone(ZoneInfo("UTC")),
+        manifold_user_id=get_authenticated_user().id,
+    )
+    st.set_page_config(layout="wide")  # Best viewed with a wide screen
+    st.title(f"Monitoring Agent: '{agent.name}'")
+    monitor_agent(agent)

prediction_market_agent_tooling/markets/data_models.py

@@ -1,8 +1,10 @@
+from datetime import datetime
 from decimal import Decimal
 from enum import Enum
 import typing as t
 from pydantic import BaseModel
 from web3 import Web3
+
 from prediction_market_agent_tooling.gtypes import (
     USD,
     HexAddress,
@@ -13,7 +15,6 @@
     xDai,
     Wei,
 )
-from datetime import datetime
 
 
 class Currency(str, Enum):
@@ -26,6 +27,28 @@ class BetAmount(BaseModel):
     currency: Currency
 
 
+class ProfitAmount(BaseModel):
+    amount: Decimal
+    currency: Currency
+
+
+class Bet(BaseModel):
+    amount: BetAmount
+    outcome: bool
+    created_time: datetime
+    market_question: str
+
+
+class ResolvedBet(Bet):
+    market_outcome: bool
+    resolved_time: datetime
+    profit: ProfitAmount
+
+    @property
+    def is_correct(self) -> bool:
+        return self.outcome == self.market_outcome
+
+
 class AgentMarket(BaseModel):
     """
     Common market class that can be created from vendor specific markets.
@@ -116,7 +139,7 @@ class ManifoldPool(BaseModel):
 
 class ManifoldMarket(BaseModel):
     """
-    https://manifold.markets
+    https://docs.manifold.markets/api#get-v0markets
     """
 
     BET_AMOUNT_CURRENCY: Currency = Currency.Mana
@@ -130,8 +153,10 @@ class ManifoldMarket(BaseModel):
     creatorName: str
     creatorUsername: str
     isResolved: bool
+    resolution: t.Optional[str] = None
+    resolutionTime: t.Optional[datetime] = None
     lastBetTime: datetime
-    lastCommentTime: t.Optional[datetime] = None  # Not always present
+    lastCommentTime: t.Optional[datetime] = None
     lastUpdatedTime: datetime
     mechanism: str
     outcomeType: str
@@ -160,3 +185,96 @@ def to_agent_market(self) -> "AgentMarket":
 
     def __repr__(self) -> str:
         return f"Manifold's market: {self.question}"
+
+
+class ProfitCached(BaseModel):
+    daily: Mana
+    weekly: Mana
+    monthly: Mana
+    allTime: Mana
+
+
+class ManifoldUser(BaseModel):
+    """
+    https://docs.manifold.markets/api#get-v0userusername
+    """
+
+    id: str
+    createdTime: datetime
+    name: str
+    username: str
+    url: str
+    avatarUrl: t.Optional[str] = None
+    bio: t.Optional[str] = None
+    bannerUrl: t.Optional[str] = None
+    website: t.Optional[str] = None
+    twitterHandle: t.Optional[str] = None
+    discordHandle: t.Optional[str] = None
+    isBot: t.Optional[bool] = None
+    isAdmin: t.Optional[bool] = None
+    isTrustworthy: t.Optional[bool] = None
+    isBannedFromPosting: t.Optional[bool] = None
+    userDeleted: t.Optional[bool] = None
+    balance: Mana
+    totalDeposits: Mana
+    lastBetTime: t.Optional[datetime] = None
+    currentBettingStreak: t.Optional[int] = None
+    profitCached: ProfitCached
+
+
+class ManifoldBetFills(BaseModel):
+    amount: Mana
+    matchedBetId: t.Optional[str]
+    shares: Decimal
+    timestamp: int
+
+
+class ManifoldBetFees(BaseModel):
+    platformFee: Decimal
+    liquidityFee: Decimal
+    creatorFee: Decimal
+
+
+class ManifoldBet(BaseModel):
+    """
+    https://docs.manifold.markets/api#get-v0bets
+    """
+
+    shares: Decimal
+    probBefore: Probability
+    isFilled: t.Optional[bool] = None
+    probAfter: Probability
+    userId: str
+    amount: Mana
+    contractId: str
+    id: str
+    fees: ManifoldBetFees
+    isCancelled: t.Optional[bool] = None
+    loanAmount: Mana
+    orderAmount: t.Optional[Mana] = None
+    fills: t.Optional[list[ManifoldBetFills]] = None
+    createdTime: datetime
+    outcome: str
+
+
+class ManifoldContractMetric(BaseModel):
+    """
+    https://docs.manifold.markets/api#get-v0marketmarketidpositions
+    """
+
+    contractId: str
+    hasNoShares: bool
+    hasShares: bool
+    hasYesShares: bool
+    invested: Decimal
+    loan: Decimal
+    maxSharesOutcome: t.Optional[str]
+    payout: Decimal
+    profit: Decimal
+    profitPercent: Decimal
+    totalShares: dict[str, Decimal]
+    userId: str
+    userUsername: str
+    userName: str
+    userAvatarUrl: str
+    lastBetTime: datetime

prediction_market_agent_tooling/markets/manifold.py

@@ -1,9 +1,17 @@
+from datetime import datetime
 import requests
 import typing as t
 from prediction_market_agent_tooling.gtypes import Mana
 from prediction_market_agent_tooling.config import APIKeys
 from prediction_market_agent_tooling.markets.data_models import (
+    ProfitAmount,
+    ResolvedBet,
+    BetAmount,
+    Currency,
+    ManifoldBet,
     ManifoldMarket,
+    ManifoldUser,
+    ManifoldContractMetric,
 )
 
 """
@@ -69,3 +77,70 @@ def place_bet(amount: Mana, market_id: str, outcome: bool) -> None:
         raise Exception(
             f"Placing bet failed: {response.status_code} {response.reason} {response.text}"
         )
+
+
+def get_authenticated_user() -> ManifoldUser:
+    url = "https://api.manifold.markets/v0/me"
+    headers = {
+        "Authorization": f"Key {APIKeys().manifold_api_key}",
+        "Content-Type": "application/json",
+    }
+    response = requests.get(url, headers=headers)
+    response.raise_for_status()
+    return ManifoldUser.model_validate(response.json())
+
+
+def get_manifold_market(market_id: str) -> ManifoldMarket:
+    url = f"https://api.manifold.markets/v0/market/{market_id}"
+    response = requests.get(url)
+    response.raise_for_status()
+    return ManifoldMarket.model_validate(response.json())
+
+
+def get_resolved_manifold_bets(
+    user_id: str,
+    start_time: datetime,
+    end_time: t.Optional[datetime],
+) -> list[ManifoldBet]:
+    url = "https://api.manifold.markets/v0/bets"
+
+    params: dict[str, str] = {"userId": user_id}
+    response = requests.get(url, params=params)
+    response.raise_for_status()
+    bets = [ManifoldBet.model_validate(x) for x in response.json()]
+    bets = [b for b in bets if b.createdTime >= start_time]
+    if end_time:
+        bets = [b for b in bets if b.createdTime < end_time]
+    bets = [b for b in bets if get_manifold_market(b.contractId).isResolved]
+    return bets
+
+
+def manifold_to_generic_resolved_bet(bet: ManifoldBet) -> ResolvedBet:
+    market = get_manifold_market(bet.contractId)
+    if not market.isResolved:
+        raise ValueError(f"Market {market.id} is not resolved.")
+    if not market.resolutionTime:
+        raise ValueError(f"Market {market.id} has no resolution time.")
+
+    # Get the profit for this bet from the corresponding position
+    positions = get_market_positions(market.id, bet.userId)
+    bet_position = next(p for p in positions if p.contractId == bet.contractId)
+    profit = bet_position.profit
+
+    return ResolvedBet(
+        amount=BetAmount(amount=bet.amount, currency=Currency.Mana),
+        outcome=bet.outcome == "YES",
+        created_time=bet.createdTime,
+        market_question=market.question,
+        market_outcome=market.resolution == "YES",
+        resolved_time=market.resolutionTime,
+        profit=ProfitAmount(amount=profit, currency=Currency.Mana),
+    )
+
+
+def get_market_positions(market_id: str, user_id: str) -> list[ManifoldContractMetric]:
+    url = f"https://api.manifold.markets/v0/market/{market_id}/positions"
+    params = {"userId": user_id}
+    response = requests.get(url, params=params)
+    response.raise_for_status()
+    return [ManifoldContractMetric.model_validate(x) for x in response.json()]

prediction_market_agent_tooling/monitor/markets/manifold.py

@@ -0,0 +1,18 @@
+from prediction_market_agent_tooling.markets.data_models import ResolvedBet
+from prediction_market_agent_tooling.markets.manifold import (
+    get_resolved_manifold_bets,
+    manifold_to_generic_resolved_bet,
+)
+from prediction_market_agent_tooling.monitor.monitor import DeployedAgent
+
+
+class DeployedManifoldAgent(DeployedAgent):
+    manifold_user_id: str
+
+    def get_resolved_bets(self) -> list[ResolvedBet]:
+        manifold_bets = get_resolved_manifold_bets(
+            user_id=self.manifold_user_id,
+            start_time=self.start_time,
+            end_time=None,
+        )
+        return [manifold_to_generic_resolved_bet(b) for b in manifold_bets]

prediction_market_agent_tooling/monitor/monitor.py

@@ -0,0 +1,64 @@
+import altair as alt
+from datetime import datetime
+from pydantic import BaseModel
+import pandas as pd
+import streamlit as st
+import typing as t
+
+from prediction_market_agent_tooling.markets.data_models import ResolvedBet
+
+
+class DeployedAgent(BaseModel):
+    name: str
+    start_time: datetime = datetime.utcnow()
+    end_time: t.Optional[datetime] = None
+
+    def get_resolved_bets(self) -> list[ResolvedBet]:
+        raise NotImplementedError("Subclasses must implement this method.")
+
+
+def monitor_agent(agent: DeployedAgent) -> None:
+    agent_bets = agent.get_resolved_bets()
+    bets_info = {
+        "Market Question": [bet.market_question for bet in agent_bets],
+        "Bet Amount": [bet.amount.amount for bet in agent_bets],
+        "Bet Outcome": [bet.outcome for bet in agent_bets],
+        "Created Time": [bet.created_time for bet in agent_bets],
+        "Resolved Time": [bet.resolved_time for bet in agent_bets],
+        "Is Correct": [bet.is_correct for bet in agent_bets],
+        "Profit": [bet.profit.amount for bet in agent_bets],
+    }
+    bets_df = pd.DataFrame(bets_info).sort_values(by="Resolved Time")
+
+    # Metrics
+    col1, col2 = st.columns(2)
+    col1.metric(label="Number of bets", value=f"{len(agent_bets)}")
+    col2.metric(label="% Correct", value=f"{100 * bets_df['Is Correct'].mean():.2f}%")
+
+    # Chart of cumulative profit per day
+    profit_info = {
+        "Time": bets_df["Resolved Time"],
+        "Cumulative Profit": bets_df["Profit"].astype(float),
+    }
+    profit_df = pd.DataFrame(profit_info)
+    profit_df["Date"] = pd.to_datetime(profit_df["Time"].dt.date)
+    profit_df = (
+        profit_df.groupby("Date")["Cumulative Profit"].sum().cumsum().reset_index()
+    )
+    profit_df["Cumulative Profit"] = profit_df["Cumulative Profit"].astype(float)
+    st.empty()
+    st.altair_chart(
+        alt.Chart(profit_df)
+        .mark_line()
+        .encode(
+            x=alt.X("Date", axis=alt.Axis(format="%Y-%m-%d"), title=None),
+            y=alt.Y("Cumulative Profit", axis=alt.Axis(format=".2f")),
+        )
+        .interactive(),
+        use_container_width=True,
+    )
+
+    # Table of resolved bets
+    st.empty()
+    st.subheader("Resolved Bet History")
+    st.table(bets_df)

pyproject.toml

@@ -10,7 +10,7 @@ buy_omen = "scripts.bet_omen:buy"
 sell_omen = "scripts.bet_omen:sell"
 
 [tool.poetry.dependencies]
-python = ">=3.9,<3.12"
+python = ">=3.10,<3.12"
 typer = "^0.9.0"
 types-requests = "^2.31.0.20240106"
 google-cloud-functions = "^1.16.0"
@@ -22,6 +22,7 @@ web3 = "^6.15.1"
 eth-typing = "^4.0.0"
 pydantic-settings = "^2.1.0"
 numpy = "^1.26.4"
+streamlit = "^1.31.0"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "*"