Repository navigation
Expand file tree
/
Copy pathccpool_proxy.py
More file actions
343 lines (298 loc) · 14.1 KB
/
Copy pathccpool_proxy.py
File metadata and controls
343 lines (298 loc) · 14.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
#!/usr/bin/env python3
"""ccpool-proxy -- rotate accounts under a LIVE Claude Code session.
ccpool proxy # listens on 127.0.0.1:8790
ANTHROPIC_BASE_URL=http://127.0.0.1:8790 claude
A running `claude` process reads its *environment* once, at startup, so
ANTHROPIC_AUTH_TOKEN cannot rotate a live session. (It does re-read the keychain
per request -- logging into a new account in one window visibly switches every
other open window -- so a keychain swapper is a real alternative. It mutates
your actual login state, which this does not.)
On a rate-limit rejection the proxy benches that account, picks the next one,
and **replays the same request** upstream. The session sees one slightly slow
response and carries on. No restart, nobody at the keyboard.
Why a proxy is right here and was wrong before
----------------------------------------------
Earlier drafts used a proxy as an *observation point*, to read rate-limit
headers off the wire. Claude Code already puts that on stdout, so the proxy was
deleted -- correctly. This is an *interception point*, and stdout cannot be one.
It also happens to fix the problem that broke every stdout-based design. This
process holds the connection to api.anthropic.com itself, so the
`anthropic-ratelimit-unified-*` headers it reads are authentic. The C1
counterexample -- an intermediary synthesising a 429 that looks exactly like
real exhaustion -- cannot apply to the one hop that has no intermediary. No
confirmation probe is needed, and utilization arrives free on every response.
Retry is only safe before any body bytes have gone downstream. A rate-limit 429
arrives as a complete response before streaming starts, so that holds; a failure
mid-stream is not retried and is passed through.
"""
from __future__ import annotations
import http.client
import http.server
import json
import os
import socketserver
import sys
import threading
import time
import urllib.parse
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import ccpool as C # noqa: E402
import ccpool_watch as W # noqa: E402
UPSTREAM = os.environ.get("CCPOOL_UPSTREAM", "https://api.anthropic.com")
PREFIX = "anthropic-ratelimit-unified-"
# Hop-by-hop headers, plus the ones we must own rather than forward.
STRIP = {"connection", "keep-alive", "proxy-authenticate", "proxy-authorization",
"te", "trailers", "transfer-encoding", "upgrade", "host",
"authorization", "x-api-key", "content-length"}
LOCK = threading.Lock()
# Idle upstream connections, per account. Without this every request pays a
# fresh TLS handshake to api.anthropic.com -- ~100-300ms on a call whose whole
# round trip is under a second. Keyed by account rather than shared, so a
# rotation never hands one org's connection to another; there is one active
# account at a time anyway, so the pool stays warm regardless.
POOL: dict[str, list] = {}
POOL_LOCK = threading.Lock()
POOL_MAX = 8
def upstream_conn():
u = urllib.parse.urlsplit(UPSTREAM)
cls = (http.client.HTTPSConnection if u.scheme == "https"
else http.client.HTTPConnection)
return cls(u.hostname, u.port, timeout=900)
def checkout(name: str):
"""An idle connection for this account, or a new one."""
with POOL_LOCK:
idle = POOL.get(name) or []
while idle:
conn = idle.pop()
if conn.sock is not None:
return conn, True
conn.close()
return upstream_conn(), False
def checkin(name: str, conn) -> None:
if conn.sock is None:
conn.close()
return
with POOL_LOCK:
idle = POOL.setdefault(name, [])
if len(idle) >= POOL_MAX:
conn.close()
return
idle.append(conn)
def record_utilization(name: str, headers) -> dict[str, dict]:
"""Every response carries the account's real quota. Free, and authentic:
there is no intermediary between here and Anthropic to forge it."""
hdrs = {k.lower(): v for k, v in headers.items()
if k.lower().startswith(PREFIX)}
limits: dict[str, dict] = {}
for key, val in hdrs.items():
rest = key[len(PREFIX):]
for field in ("-status", "-utilization", "-reset"):
if rest.endswith(field):
abbrev = rest[: -len(field)]
if not abbrev or abbrev in ("overage", "fallback"):
continue # overage-status reads "rejected" when healthy
claim = C.ABBREV.get(abbrev)
if claim:
limits.setdefault(claim, {})[field[1:]] = val
return limits
class Handler(http.server.BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
server_version = "ccpool"
def log_message(self, fmt, *a):
pass
def note(self, msg):
# Timestamped: without one, a log line cannot be correlated with the
# slowdown a human noticed, which is exactly what it is read for.
print(f"{time.strftime('%H:%M:%S')} ccpool-proxy: {msg}",
file=sys.stderr, flush=True)
def do_POST(self):
self.relay()
def do_GET(self):
self.relay()
def relay(self):
length = int(self.headers.get("content-length") or 0)
body = self.rfile.read(length) if length else b""
model = self.model_of(body)
fwd = {k: v for k, v in self.headers.items() if k.lower() not in STRIP}
tried: set[str] = set()
for _ in range(max(1, len(C.accounts()))):
with LOCK:
name = C.pick(model, tried)
if name is None:
return self.no_accounts(model)
tried.add(name)
token = C.accounts()[name].get("token", "")
try:
status, headers, resp, conn = self.send_upstream(
fwd, body, token, name)
except OSError as e:
self.note(f"{name}: upstream unreachable ({e.__class__.__name__})")
return self.fail(502, "upstream unreachable")
limits = record_utilization(name, headers)
with LOCK:
C.remember_last(name)
if status == 429:
spent = C.exhausted_claims(limits)
detail = (", ".join(f"{c}={r.get('status','?')}/"
f"{r.get('utilization','?')}"
for c, r in sorted(limits.items()))
or "NO unified rate-limit headers")
retry = headers.get("retry-after")
try:
wait = int(retry or 0)
except ValueError:
wait = 0
reason = "reported exhausted quota" if spent else "no exhausted quota"
if not spent:
try:
snippet = resp.read(200).decode("utf-8", "replace")
except OSError:
snippet = "<unreadable>"
if wait > 3600:
# This authentic cooldown belongs to the requested
# model's weekly claim, not every model. Unknown model
# names keep the old conservative global fallback.
known_model = model and any(k in model for k in W.CLAIMS)
claim = W.claims_for(model)[-1] if known_model else "five_hour"
spent = {claim: int(time.time()) + wait}
reason = "long Retry-After for requested model"
else:
reason += f"; body={snippet!r}"
chosen = ",".join(sorted(spent)) if spent else "none"
self.note(f"{name}: 429 model={model!r}; quota={detail}; "
f"retry-after={retry or '?'}; cooldown={chosen}; "
f"reason={reason}")
conn.close()
if spent:
with LOCK:
for claim, reset in spent.items():
C.bench(name, claim, reset)
self.note(f"{name} out on {','.join(sorted(spent))} -- "
f"rotating to another account and retrying request")
continue
# A 429 carrying no spent claim is upstream load, not quota.
# Rotating would not help and benching would be a lie.
return self.fail(429, "upstream rate limit (not account quota)")
# Retire before streaming, not after: the body can take a minute to
# drain, and a request arriving in that window would otherwise pick
# an account already known to be empty.
self.retire_if_full(name, limits)
self.stream_down(status, headers, resp, conn, name)
return
self.no_accounts(model)
def send_upstream(self, fwd, body, token, name):
headers = dict(fwd)
headers["Authorization"] = f"Bearer {token}"
if body:
headers["Content-Length"] = str(len(body))
# A pooled connection can have been closed by the far end since we last
# used it, and there is no way to know until the write fails. So: try
# it, and on failure fall back to a fresh one exactly once. Replaying is
# safe for the same reason the rotation retry is -- nothing has gone
# downstream yet, and a request that died on send was never processed.
for reused in (True, False):
conn, was_pooled = (checkout(name) if reused else (upstream_conn(), False))
try:
conn.request(self.command, self.path, body=body, headers=headers)
resp = conn.getresponse()
return resp.status, resp.headers, resp, conn
except (http.client.HTTPException, OSError):
conn.close()
if not (reused and was_pooled):
raise # a brand new connection failing is a real error
def stream_down(self, status, headers, resp, conn, name):
"""Past this point nothing may be retried -- bytes are on the wire."""
done = False
try:
self.send_response(status)
for k, v in headers.items():
if k.lower() in ("connection", "transfer-encoding",
"content-length", "keep-alive"):
continue
self.send_header(k, v)
self.send_header("Connection", "close")
self.send_header("Transfer-Encoding", "chunked")
self.end_headers()
while True:
chunk = resp.read(8192)
if not chunk:
break
self.wfile.write(b"%X\r\n%s\r\n" % (len(chunk), chunk))
self.wfile.flush()
self.wfile.write(b"0\r\n\r\n")
self.wfile.flush()
done = True
except (BrokenPipeError, ConnectionResetError):
pass
finally:
# Only a connection drained to the end of its body can be reused.
# A client that hung up mid-stream leaves unread bytes on the
# socket, and handing that to the next request would splice one
# response into another.
if done and resp.isclosed() and not resp.will_close:
checkin(name, conn)
else:
conn.close()
def retire_if_full(self, name, limits):
"""Bench an account the moment a SUCCESSFUL response says it is full.
This is the whole answer to "rotate early and waste the tail, or rotate
late and eat a failed request". Neither: the response that consumed the
last of the quota also reports the quota is gone, so the next request
picks a different account and no request is ever rejected. Zero forfeit,
zero wasted round trip.
"""
spent = C.exhausted_claims(limits)
with LOCK:
C.record_resets(name, limits)
for claim, reset in spent.items():
C.bench(name, claim, reset)
if spent:
self.note(f"{name} now full on {','.join(sorted(spent))} -- retired "
f"before it could reject anything")
def model_of(self, body):
try:
return json.loads(body).get("model")
except Exception: # noqa: BLE001
return None
def no_accounts(self, model):
with LOCK:
when = C.next_reset(model)
hint = (f" Earliest reset {time.strftime('%a %H:%M', time.localtime(when))}."
if when else "")
self.note(f"every account is benched.{hint}")
self.fail(429, f"all pooled accounts are rate limited.{hint}")
def fail(self, status, message):
payload = json.dumps({"type": "error",
"error": {"type": "rate_limit_error" if status == 429
else "api_error",
"message": f"ccpool: {message}"}}).encode()
try:
self.send_response(status)
self.send_header("content-type", "application/json")
self.send_header("content-length", str(len(payload)))
self.send_header("Connection", "close")
self.end_headers()
self.wfile.write(payload)
except (BrokenPipeError, ConnectionResetError):
pass
class Server(socketserver.ThreadingMixIn, http.server.HTTPServer):
daemon_threads = True
allow_reuse_address = True
def serve(host="127.0.0.1", port=8790):
if not C.accounts():
sys.exit("ccpool: no accounts. `ccpool add <name>` first.")
srv = Server((host, port), Handler)
print(f"ccpool-proxy on http://{host}:{port} -> {UPSTREAM}\n"
f" {len(C.accounts())} accounts. Point Claude Code at it:\n"
f" export ANTHROPIC_BASE_URL=http://{host}:{port}\n"
f" ...or, behind headroom, ANTHROPIC_TARGET_API_URL (headroom owns\n"
f" ANTHROPIC_BASE_URL). ccpool must stay the last hop before\n"
f" Anthropic so its rate-limit headers have no intermediary.",
file=sys.stderr)
try:
srv.serve_forever()
except KeyboardInterrupt:
pass
if __name__ == "__main__":
serve(port=int(sys.argv[1]) if len(sys.argv) > 1 else 8790)