From 2374ef137d4f9da0104ccb4759882adfa452810d Mon Sep 17 00:00:00 2001 From: Chris Olson Date: Wed, 24 Jun 2026 10:19:52 -0400 Subject: [PATCH] Add OpenAI-compatible LLM provider (URL/model/key configurable) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - New provider 'openai' calls any OpenAI-compatible /chat/completions endpoint (OpenAI, OpenRouter, Together, Groq, NVIDIA, vLLM, Ollama, ...), set via OPENAI_BASE_URL / OPENAI_API_KEY / OPENAI_MODEL. Now the default; auto-detects. - Robustness for models that degenerate (e.g. Kimi repetition loops): anti- repetition sampling (temperature + frequency penalty), and a schema-aware retry — call_json(require_keys=...) retries a fresh sample on degenerate or unparseable output (LLMRetryable). Relevance requires 'signals', synth 'briefs'. - response_format=json_object (toggle via OPENAI_JSON_MODE); base URL accepts a full /chat/completions URL or a /v1 base. Co-Authored-By: Claude Opus 4.8 --- sales-agent/.env.example | 16 +++-- sales-agent/README.md | 16 +++-- sales-agent/src/agents/relevance.py | 2 +- sales-agent/src/agents/synthesizer.py | 2 +- sales-agent/src/config.py | 37 ++++++++++-- sales-agent/src/llm.py | 84 ++++++++++++++++++++++++--- 6 files changed, 135 insertions(+), 22 deletions(-) diff --git a/sales-agent/.env.example b/sales-agent/.env.example index 22453e2..a97ad98 100644 --- a/sales-agent/.env.example +++ b/sales-agent/.env.example @@ -1,12 +1,20 @@ # Copy to .env and fill in. The loop runs WITHOUT a key using a mock reasoner. # --- LLM provider --- -# LLM_PROVIDER: gemini | anthropic (auto-detected from whichever key is set, -# Gemini preferred, if left blank). Provider with a missing key => mock reasoner. -LLM_PROVIDER=gemini +# LLM_PROVIDER: openai | gemini | anthropic (auto-detected from whichever key is +# set if blank; OpenAI-compatible preferred). Provider w/ a missing key => mock. +LLM_PROVIDER=openai + +# OpenAI-compatible endpoint — works with OpenAI, OpenRouter, Together, Groq, +# vLLM, Ollama, LM Studio, etc. Just point these three at any /chat/completions API: +OPENAI_BASE_URL=https://api.openai.com/v1 +OPENAI_API_KEY= +OPENAI_MODEL=gpt-4o-mini +OPENAI_JSON_MODE=1 # set 0 if the endpoint rejects response_format=json_object + +# Other providers: GEMINI_API_KEY= GEMINI_MODEL=gemini-2.5-flash - ANTHROPIC_API_KEY= LLM_MODEL=claude-sonnet-4-6 diff --git a/sales-agent/README.md b/sales-agent/README.md index 8e1db2b..2b87892 100644 --- a/sales-agent/README.md +++ b/sales-agent/README.md @@ -56,7 +56,14 @@ The relevance filter and synthesizer call whichever provider is configured. Set `.env`: ```bash -# Gemini (current default) +# OpenAI-compatible (default) — OpenAI, OpenRouter, Together, Groq, vLLM, Ollama, ... +LLM_PROVIDER=openai +OPENAI_BASE_URL=https://api.openai.com/v1 # any /chat/completions endpoint +OPENAI_API_KEY=... +OPENAI_MODEL=gpt-4o-mini +OPENAI_JSON_MODE=1 # set 0 if the endpoint rejects json_object + +# or Gemini LLM_PROVIDER=gemini GEMINI_API_KEY=... GEMINI_MODEL=gemini-2.5-flash @@ -67,9 +74,10 @@ ANTHROPIC_API_KEY=... LLM_MODEL=claude-sonnet-4-6 ``` -If `LLM_PROVIDER` is blank it auto-detects from whichever key is present (Gemini -preferred). With no key, a deterministic mock reasoner produces grounded briefs so -the full pipeline still runs offline. Both providers use plain `urllib` — no SDKs. +If `LLM_PROVIDER` is blank it auto-detects from whichever key is present (OpenAI- +compatible preferred). With no key, a deterministic mock reasoner produces grounded +briefs so the full pipeline still runs offline. All providers use plain `urllib` — +no SDKs. LLM calls are batched (one relevance call + one synthesis call per account). ## Exa.ai news connector & free-plan protection diff --git a/sales-agent/src/agents/relevance.py b/sales-agent/src/agents/relevance.py index 2b73958..55ea46a 100644 --- a/sales-agent/src/agents/relevance.py +++ b/sales-agent/src/agents/relevance.py @@ -131,7 +131,7 @@ def _evaluate_batch(prepared, account) -> list[tuple[NormalizedSignal, Relevance signals_block=_signals_block(prepared), ) try: - data = llm.call_json(SYSTEM, user, max_tokens=2200) + data = llm.call_json(SYSTEM, user, max_tokens=2200, require_keys={"signals"}) except llm.LLMError as e: print(f" [relevance] batched LLM error, falling back to mock: {e}") return _mock_batch(prepared, account) diff --git a/sales-agent/src/agents/synthesizer.py b/sales-agent/src/agents/synthesizer.py index 1593b9a..11aa6bd 100644 --- a/sales-agent/src/agents/synthesizer.py +++ b/sales-agent/src/agents/synthesizer.py @@ -113,7 +113,7 @@ def _batched_bodies(groups: list[tuple[str, dict, list]], account: dict) -> dict capability_blocks="\n\n".join(blocks), ) try: - data = llm.call_json(SYSTEM, user, max_tokens=4096) + data = llm.call_json(SYSTEM, user, max_tokens=4096, require_keys={"briefs"}) except llm.LLMError as e: print(f" [synthesize] batched LLM error, falling back to mock: {e}") return {} diff --git a/sales-agent/src/config.py b/sales-agent/src/config.py index abed59d..ec29b27 100644 --- a/sales-agent/src/config.py +++ b/sales-agent/src/config.py @@ -28,9 +28,31 @@ def _load_dotenv() -> None: _load_dotenv() # --- LLM --- -# Provider abstraction: gemini | anthropic | mock. If LLM_PROVIDER is unset we -# auto-detect from whichever key is present (Gemini preferred), falling back to -# a deterministic mock reasoner so the loop still runs offline. +# Provider abstraction: openai | gemini | anthropic | mock. If LLM_PROVIDER is +# unset we auto-detect from whichever key is present (OpenAI-compatible first), +# falling back to a deterministic mock reasoner so the loop still runs offline. +# +# "openai" is any OpenAI-compatible /chat/completions endpoint — OpenAI, OpenRouter, +# Together, Groq, vLLM, Ollama, LM Studio, etc. — configured by three env vars: +# OPENAI_BASE_URL (e.g. https://api.openai.com/v1) +# OPENAI_API_KEY +# OPENAI_MODEL (e.g. gpt-4o-mini) +OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY", "").strip() +OPENAI_BASE_URL = os.environ.get("OPENAI_BASE_URL", "https://api.openai.com/v1").strip().rstrip("/") +OPENAI_MODEL = os.environ.get("OPENAI_MODEL", "gpt-4o-mini").strip() +# Some endpoints/models don't support response_format=json_object — turn off if so. +OPENAI_JSON_MODE = os.environ.get("OPENAI_JSON_MODE", "1").strip() == "1" +# Sampling/anti-repetition. A small frequency penalty + non-trivial temperature +# avoids the repetition-loop degeneration some models (e.g. Kimi) fall into at +# very low temperature. Penalties are omitted from the request when set to 0 +# (for endpoints that reject them). +OPENAI_TEMPERATURE = float(os.environ.get("OPENAI_TEMPERATURE", "0.5")) +OPENAI_FREQUENCY_PENALTY = float(os.environ.get("OPENAI_FREQUENCY_PENALTY", "0.4")) +OPENAI_PRESENCE_PENALTY = float(os.environ.get("OPENAI_PRESENCE_PENALTY", "0.0")) +# Retries when a model returns degenerate/unparseable content (stochastic — a +# retry usually succeeds). Distinct from the network/429 backoff. +LLM_CONTENT_RETRIES = int(os.environ.get("LLM_CONTENT_RETRIES", "3")) + ANTHROPIC_API_KEY = os.environ.get("ANTHROPIC_API_KEY", "").strip() LLM_MODEL = os.environ.get("LLM_MODEL", "claude-sonnet-4-6") @@ -39,7 +61,9 @@ GEMINI_MODEL = os.environ.get("GEMINI_MODEL", "gemini-2.5-flash") _provider = os.environ.get("LLM_PROVIDER", "").strip().lower() if not _provider: - if GEMINI_API_KEY: + if OPENAI_API_KEY: + _provider = "openai" + elif GEMINI_API_KEY: _provider = "gemini" elif ANTHROPIC_API_KEY: _provider = "anthropic" @@ -48,6 +72,8 @@ if not _provider: # A forced USE_MOCK_LLM=1, or selecting a provider whose key is missing, => mock. if os.environ.get("USE_MOCK_LLM", "").strip() == "1": _provider = "mock" +elif _provider == "openai" and not OPENAI_API_KEY: + _provider = "mock" elif _provider == "gemini" and not GEMINI_API_KEY: _provider = "mock" elif _provider == "anthropic" and not ANTHROPIC_API_KEY: @@ -60,6 +86,9 @@ USE_MOCK_LLM = LLM_PROVIDER == "mock" def llm_label() -> str: if LLM_PROVIDER == "mock": return "MOCK reasoner (no LLM key)" + if LLM_PROVIDER == "openai": + host = OPENAI_BASE_URL.split("//")[-1].split("/")[0] + return f"OpenAI-compatible ({OPENAI_MODEL} @ {host})" if LLM_PROVIDER == "gemini": return f"Gemini ({GEMINI_MODEL})" return f"Anthropic ({LLM_MODEL})" diff --git a/sales-agent/src/llm.py b/sales-agent/src/llm.py index 9fe9905..fd702ed 100644 --- a/sales-agent/src/llm.py +++ b/sales-agent/src/llm.py @@ -22,6 +22,10 @@ class LLMError(RuntimeError): pass +class LLMRetryable(LLMError): + """A failure worth retrying with a fresh sample (degenerate/unparseable output).""" + + def _extract_json(text: str) -> dict: """Pull the first JSON object out of a model response.""" text = text.strip() @@ -32,13 +36,12 @@ def _extract_json(text: str) -> dict: start = text.find("{") end = text.rfind("}") if start == -1 or end == -1: - raise LLMError(f"No JSON object found in LLM output: {text[:200]}") + raise LLMRetryable(f"No JSON object found in LLM output: {text[:200]}") try: return json.loads(text[start : end + 1]) except json.JSONDecodeError as e: - # Wrap so callers' `except LLMError` can fall back to mock gracefully - # (e.g. on a truncated/malformed response). - raise LLMError(f"Malformed JSON from LLM ({e}): ...{text[max(0, end-120):end + 1]}") + # Retryable: a fresh sample usually parses (truncated/malformed output). + raise LLMRetryable(f"Malformed JSON from LLM ({e}): ...{text[max(0, end-120):end + 1]}") def _post(url: str, payload: dict, headers: dict, provider: str) -> dict: @@ -66,6 +69,41 @@ def _post(url: str, payload: dict, headers: dict, provider: str) -> dict: raise LLMError(f"{provider} API still rate-limited after retries") +def _call_openai(system: str, user: str, max_tokens: int) -> dict: + base = config.OPENAI_BASE_URL + url = base if base.endswith("/chat/completions") else f"{base}/chat/completions" + payload = { + "model": config.OPENAI_MODEL, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + "max_tokens": max_tokens, + "temperature": config.OPENAI_TEMPERATURE, + } + if config.OPENAI_FREQUENCY_PENALTY: + payload["frequency_penalty"] = config.OPENAI_FREQUENCY_PENALTY + if config.OPENAI_PRESENCE_PENALTY: + payload["presence_penalty"] = config.OPENAI_PRESENCE_PENALTY + if config.OPENAI_JSON_MODE: + payload["response_format"] = {"type": "json_object"} + headers = {"authorization": f"Bearer {config.OPENAI_API_KEY}"} + body = _post(url, payload, headers, "OpenAI-compatible") + choices = body.get("choices", []) + if not choices: + raise LLMError(f"OpenAI-compatible API returned no choices: {json.dumps(body)[:300]}") + choice = choices[0] + text = (choice.get("message", {}).get("content") or "").strip() + finish = choice.get("finish_reason") + # Some models degenerate into a repetition loop (finish_reason='repetition') + # or return empty content — retry with a fresh sample rather than parse junk. + if finish == "repetition": + raise LLMRetryable("degenerate response (repetition loop)") + if not text: + raise LLMRetryable("empty response content") + return _extract_json(text) + + def _call_gemini(system: str, user: str, max_tokens: int) -> dict: url = ( "https://generativelanguage.googleapis.com/v1beta/models/" @@ -108,15 +146,45 @@ def _call_anthropic(system: str, user: str, max_tokens: int) -> dict: return _extract_json(text) -def call_json(system: str, user: str, max_tokens: int = 1500) -> dict: - """Single-turn LLM call that returns parsed JSON.""" - if config.USE_MOCK_LLM: - raise _MockSignal() # callers catch this and run their mock path +def _dispatch(system: str, user: str, max_tokens: int) -> dict: + if config.LLM_PROVIDER == "openai": + return _call_openai(system, user, max_tokens) if config.LLM_PROVIDER == "gemini": return _call_gemini(system, user, max_tokens) return _call_anthropic(system, user, max_tokens) +def call_json( + system: str, user: str, max_tokens: int = 1500, require_keys: set[str] | None = None +) -> dict: + """Single-turn LLM call that returns parsed JSON. + + Retries on degenerate/unparseable output (LLMRetryable) with a fresh sample; + network/HTTP errors and 429s are handled in _post. If `require_keys` is given, + a response missing all of those top-level keys is treated as degenerate and + retried (catches valid-but-empty junk like `{}` from a repetition loop). + After retries are exhausted the error propagates and callers fall back to mock. + """ + if config.USE_MOCK_LLM: + raise _MockSignal() # callers catch this and run their mock path + attempts = max(1, config.LLM_CONTENT_RETRIES + 1) + for i in range(attempts): + try: + data = _dispatch(system, user, max_tokens) + if require_keys and not any(k in data for k in require_keys): + raise LLMRetryable( + f"response missing required key(s) {sorted(require_keys)}; " + f"got keys {list(data)[:6]}" + ) + return data + except LLMRetryable as e: + if i < attempts - 1: + print(f" [llm] retrying degenerate/unparseable response " + f"({i + 1}/{attempts - 1}): {str(e)[:100]}") + continue + raise + + class _MockSignal(Exception): """Internal sentinel: tells the caller to use its deterministic mock branch."""