247 lines
7.6 KiB
Python
247 lines
7.6 KiB
Python
import json
|
|
import time
|
|
import sqlite3
|
|
import requests
|
|
from typing import Any, Dict, Optional, Tuple
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from requests.adapters import HTTPAdapter
|
|
from urllib3.util.retry import Retry
|
|
|
|
GAMMA = "https://gamma-api.polymarket.com"
|
|
CLOB = "https://clob.polymarket.com"
|
|
INTERVAL_SEC = 15 * 60 # 900
|
|
|
|
|
|
def make_session() -> requests.Session:
|
|
s = requests.Session()
|
|
s.headers.update({"User-Agent": "Mozilla/5.0"})
|
|
retry = Retry(
|
|
total=5,
|
|
backoff_factor=0.4,
|
|
status_forcelist=(429, 500, 502, 503, 504),
|
|
allowed_methods=("GET",),
|
|
raise_on_status=False,
|
|
)
|
|
adapter = HTTPAdapter(max_retries=retry, pool_connections=20, pool_maxsize=20)
|
|
s.mount("https://", adapter)
|
|
s.mount("http://", adapter)
|
|
return s
|
|
|
|
|
|
def parse_jsonish_list(v):
|
|
if v is None:
|
|
return []
|
|
if isinstance(v, list):
|
|
return v
|
|
if isinstance(v, str):
|
|
s = v.strip()
|
|
if s.startswith("[") and s.endswith("]"):
|
|
return json.loads(s)
|
|
return [x.strip() for x in s.split(",") if x.strip()]
|
|
return []
|
|
|
|
|
|
def get_market_by_slug(session: requests.Session, slug: str) -> Dict[str, Any]:
|
|
r = session.get(f"{GAMMA}/markets/slug/{slug}", timeout=20)
|
|
if r.status_code == 404:
|
|
raise FileNotFoundError(slug)
|
|
r.raise_for_status()
|
|
return r.json()
|
|
|
|
|
|
def get_price(session: requests.Session, token_id: str, side: str) -> Optional[float]:
|
|
r = session.get(f"{CLOB}/price", params={"token_id": token_id, "side": side}, timeout=20)
|
|
r.raise_for_status()
|
|
p = r.json().get("price")
|
|
return float(p) if p is not None else None
|
|
|
|
|
|
def get_book(session: requests.Session, token_id: str) -> Dict[str, Any]:
|
|
r = session.get(f"{CLOB}/book", params={"token_id": token_id}, timeout=20)
|
|
r.raise_for_status()
|
|
return r.json()
|
|
|
|
|
|
def compute_mid(bid: Optional[float], ask: Optional[float]) -> Tuple[Optional[float], Optional[float]]:
|
|
if bid is None and ask is None:
|
|
return None, None
|
|
if bid is None:
|
|
return ask, None
|
|
if ask is None:
|
|
return bid, None
|
|
return (bid + ask) / 2.0, (ask - bid)
|
|
|
|
|
|
def trim_book(book: Dict[str, Any], top_n: int = 10) -> Dict[str, Any]:
|
|
bids = book.get("bids") or []
|
|
asks = book.get("asks") or []
|
|
bids = sorted(bids, key=lambda x: float(x["price"]), reverse=True)[:top_n]
|
|
asks = sorted(asks, key=lambda x: float(x["price"]))[:top_n]
|
|
return {"bids": bids, "asks": asks}
|
|
|
|
|
|
def fetch_token_bundle(session: requests.Session, token_id: str, top_n: int = 10) -> Dict[str, Any]:
|
|
with ThreadPoolExecutor(max_workers=3) as ex:
|
|
futs = {
|
|
ex.submit(get_price, session, token_id, "buy"): "bid",
|
|
ex.submit(get_price, session, token_id, "sell"): "ask",
|
|
ex.submit(get_book, session, token_id): "book",
|
|
}
|
|
out: Dict[str, Any] = {"token_id": token_id}
|
|
for fut in as_completed(futs):
|
|
out[futs[fut]] = fut.result()
|
|
|
|
mid, spread = compute_mid(out.get("bid"), out.get("ask"))
|
|
out["mid"] = mid
|
|
out["spread"] = spread
|
|
out["book"] = trim_book(out.get("book") or {}, top_n=top_n)
|
|
return out
|
|
|
|
|
|
def current_bucket_ts(interval_sec: int = INTERVAL_SEC, mode: str = "floor") -> int:
|
|
now = int(time.time())
|
|
if mode == "ceil":
|
|
return ((now + interval_sec - 1) // interval_sec) * interval_sec
|
|
return (now // interval_sec) * interval_sec
|
|
|
|
|
|
def build_slug(prefix: str, bucket_ts: int) -> str:
|
|
return f"{prefix}-{bucket_ts}"
|
|
|
|
|
|
def fetch_updown_by_slug(session: requests.Session, slug: str, decimals: int = 0, top_n: int = 10) -> Dict[str, Any]:
|
|
m = get_market_by_slug(session, slug)
|
|
outcomes = [str(x) for x in parse_jsonish_list(m.get("outcomes"))]
|
|
token_ids = [str(x) for x in parse_jsonish_list(m.get("clobTokenIds"))]
|
|
if len(token_ids) < 2:
|
|
raise RuntimeError(f"clobTokenIds missing/too short: {token_ids}")
|
|
|
|
token_map = dict(zip(outcomes, token_ids))
|
|
up_id = token_map.get("Up") or token_map.get("Yes") or token_ids[0]
|
|
dn_id = token_map.get("Down") or token_map.get("No") or token_ids[1]
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as ex:
|
|
up = ex.submit(fetch_token_bundle, session, up_id, top_n).result()
|
|
dn = ex.submit(fetch_token_bundle, session, dn_id, top_n).result()
|
|
|
|
if up["mid"] is None or dn["mid"] is None:
|
|
return {
|
|
"error": "missing mid price",
|
|
"slug": slug,
|
|
"market_id": m.get("id"),
|
|
"question": m.get("question"),
|
|
"up": up,
|
|
"down": dn,
|
|
}
|
|
|
|
s = float(up["mid"]) + float(dn["mid"])
|
|
up_pct = round(float(up["mid"]) / s * 100, decimals)
|
|
dn_pct = round(float(dn["mid"]) / s * 100, decimals)
|
|
|
|
return {
|
|
"question": m.get("question"),
|
|
"slug": m.get("slug"),
|
|
"market_id": m.get("id"),
|
|
"up_pct": up_pct,
|
|
"down_pct": dn_pct,
|
|
"up": up,
|
|
"down": dn,
|
|
}
|
|
|
|
|
|
def fetch_current_market(
|
|
slug_prefix: str = "eth-updown-15m",
|
|
decimals: int = 0,
|
|
top_n: int = 10,
|
|
ts_mode: str = "floor",
|
|
probe_offsets: Tuple[int, ...] = (0, -INTERVAL_SEC, INTERVAL_SEC),
|
|
) -> Dict[str, Any]:
|
|
session = make_session()
|
|
base_ts = current_bucket_ts(INTERVAL_SEC, mode=ts_mode)
|
|
fetched_at = int(time.time()) # 当前抓取时间戳
|
|
|
|
last_err: Optional[Exception] = None
|
|
for off in probe_offsets:
|
|
bucket_ts = base_ts + off
|
|
slug = build_slug(slug_prefix, bucket_ts)
|
|
try:
|
|
data = fetch_updown_by_slug(session, slug, decimals=decimals, top_n=top_n)
|
|
data["bucket_ts"] = bucket_ts
|
|
data["fetched_at"] = fetched_at
|
|
return data
|
|
except FileNotFoundError as e:
|
|
last_err = e
|
|
continue
|
|
|
|
raise RuntimeError(f"Could not find market for current bucket. last_err={last_err}")
|
|
|
|
|
|
# ----------------------------
|
|
# SQLite (3 columns)
|
|
# ----------------------------
|
|
DDL = """
|
|
CREATE TABLE IF NOT EXISTS pm_snapshots
|
|
(
|
|
bucket_ts
|
|
INTEGER
|
|
NOT
|
|
NULL,
|
|
fetched_at
|
|
INTEGER
|
|
NOT
|
|
NULL,
|
|
raw_json
|
|
TEXT
|
|
NOT
|
|
NULL,
|
|
PRIMARY
|
|
KEY
|
|
(
|
|
bucket_ts,
|
|
fetched_at
|
|
)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_pm_snapshots_fetched_at ON pm_snapshots(fetched_at); \
|
|
"""
|
|
|
|
|
|
def init_db(db_path: str = "polymarket.db") -> None:
|
|
conn = sqlite3.connect(db_path)
|
|
try:
|
|
conn.execute("PRAGMA journal_mode=WAL;")
|
|
conn.executescript(DDL)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def save_snapshot(bucket_ts: int, fetched_at: int, raw: Dict[str, Any], db_path: str = "polymarket.db") -> None:
|
|
conn = sqlite3.connect(db_path)
|
|
try:
|
|
conn.execute("PRAGMA journal_mode=WAL;")
|
|
conn.execute(
|
|
"INSERT OR IGNORE INTO pm_snapshots(bucket_ts, fetched_at, raw_json) VALUES (?, ?, ?)",
|
|
(int(bucket_ts), int(fetched_at), json.dumps(raw, ensure_ascii=False)),
|
|
)
|
|
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
init_db("polymarket.db")
|
|
|
|
while True:
|
|
data = fetch_current_market(
|
|
slug_prefix="eth-updown-15m",
|
|
decimals=0,
|
|
top_n=10,
|
|
ts_mode="floor", # 如果你发现应该取下一档，改成 'ceil'
|
|
probe_offsets=(0, -900, 900), # 先试当前，再试前一档/后一档
|
|
)
|
|
|
|
save_snapshot(data["bucket_ts"], data["fetched_at"], data, "polymarket.db")
|
|
print("saved:", data["bucket_ts"], data["fetched_at"])
|