Switch GPT client to GLM-5.3-Flash

embodied_gen/utils/gpt_clients.py

@@ -29,11 +29,12 @@ from typing import Optional
 
 import openai
 import yaml
+from huggingface_hub import get_token
 from openai import AzureOpenAI, OpenAI  # pip install openai
 from PIL import Image
 from tenacity import (
     retry,
-    retry_if_not_exception_type,
+    retry_if_exception,
     stop_after_attempt,
     stop_after_delay,
     wait_random_exponential,
@@ -50,10 +51,29 @@ __all__ = [
 
 CONFIG_FILE = str(Path(__file__).with_name("gpt_config.yaml"))
 DEFAULT_GPT_TIMEOUT = float(os.environ.get("GPT_TIMEOUT", 90))
+DEFAULT_HF_ENDPOINT = "https://router.huggingface.co/v1"
+DEFAULT_HF_MODEL = "zai-org/GLM-5.3-Flash:baseten"
+HF_MAX_IMAGE_DIMENSION = int(os.environ.get("HF_MAX_IMAGE_DIMENSION", 1024))
+HF_IMAGE_JPEG_QUALITY = int(os.environ.get("HF_IMAGE_JPEG_QUALITY", 90))
+OPENAI_MAX_IMAGE_DIMENSION = int(
+    os.environ.get("OPENAI_MAX_IMAGE_DIMENSION", 1024)
+)
 # GPT-5.x counts reasoning tokens against this cap, so it must be high
 # enough to leave room for both reasoning and the visible reply.
 GPT5_DEFAULT_MAX_COMPLETION_TOKENS = 8192
+REASONING_VLM_DEFAULT_MAX_TOKENS = int(
+    os.environ.get("REASONING_VLM_MAX_TOKENS", 2048)
+)
 _CODEX_DEFAULT_REASONING_EFFORT = "medium"
+_HF_PROVIDERS = {"hf", "huggingface"}
+_OPENAI_COMPATIBLE_PROVIDERS = {"openai", *_HF_PROVIDERS}
+_NON_RETRYABLE_API_ERRORS = (
+    openai.AuthenticationError,
+    openai.BadRequestError,
+    openai.NotFoundError,
+    openai.PermissionDeniedError,
+    openai.UnprocessableEntityError,
+)
 _CODEX_ENV_KEYS = {
     "ALL_PROXY",
     "CODEX_HOME",
@@ -77,19 +97,50 @@ def _resolve_agent_settings(
     if provider_override is not None:
         agent_config = {}
 
+    endpoint = environ.get("ENDPOINT", agent_config.get("endpoint"))
+    provider = provider_override or agent_config.get("provider")
+    provider_name = (provider or "").lower()
+    is_huggingface = provider_name in _HF_PROVIDERS or (
+        endpoint is not None and "router.huggingface.co" in endpoint
+    )
+    if is_huggingface:
+        provider = "huggingface"
+        endpoint = endpoint or DEFAULT_HF_ENDPOINT
+
+    api_key = environ.get("API_KEY")
+    if api_key is None and is_huggingface:
+        api_key = environ.get("HF_TOKEN") or environ.get(
+            "HUGGING_FACE_HUB_TOKEN"
+        ) or get_token()
+    if api_key is None:
+        api_key = agent_config.get("api_key")
+
+    api_version = environ.get("API_VERSION", agent_config.get("api_version"))
+    if is_huggingface:
+        api_version = None
+
     return {
-        "endpoint": environ.get("ENDPOINT", agent_config.get("endpoint")),
-        "api_key": environ.get("API_KEY", agent_config.get("api_key")),
-        "api_version": environ.get(
-            "API_VERSION", agent_config.get("api_version")
-        ),
-        "model_name": environ.get(
-            "MODEL_NAME", agent_config.get("model_name")
-        ),
-        "provider": provider_override or agent_config.get("provider"),
+        "endpoint": endpoint,
+        "api_key": api_key,
+        "api_version": api_version,
+        "model_name": environ.get("MODEL_NAME")
+        or agent_config.get("model_name")
+        or (DEFAULT_HF_MODEL if is_huggingface else None),
+        "provider": provider,
+        "bill_to": environ.get("HF_BILL_TO") if is_huggingface else None,
     }
 
 
+def _is_retryable_api_error(error: BaseException) -> bool:
+    """Return whether an API failure may succeed when retried."""
+    if isinstance(error, _NON_RETRYABLE_API_ERRORS):
+        return False
+    status_code = getattr(error, "status_code", None)
+    if status_code is not None and 400 <= status_code < 500:
+        return status_code in {408, 409, 429}
+    return True
+
+
 def _codex_subprocess_environment() -> dict[str, str]:
     """Return the minimal host environment required by Codex CLI."""
     return {
@@ -144,9 +195,12 @@ class GPTclient:
         check_connection (bool, optional): Whether to check API connection.
         verbose (bool, optional): Enable verbose logging.
         timeout (float, optional): Max seconds for a single GPT request.
-        provider (str, optional): Backend provider. Use ``codex`` to reuse a
-            local Codex CLI login; otherwise the existing Azure/OpenAI-
-            compatible API selection is preserved.
+        provider (str, optional): Backend provider. Use ``huggingface`` for
+            Hugging Face Inference Providers or ``codex`` to reuse a local
+            Codex CLI login; otherwise the existing Azure/OpenAI-compatible
+            API selection is preserved.
+        bill_to (str, optional): Hugging Face organization charged for routed
+            requests. Ignored by other providers.
 
     Example:
         ```sh
@@ -176,10 +230,13 @@ class GPTclient:
         verbose: bool = False,
         timeout: float = DEFAULT_GPT_TIMEOUT,
         provider: Optional[str] = None,
+        bill_to: Optional[str] = None,
     ):
         self.provider = (
             provider or ("azure" if api_version else "openai")
         ).lower()
+        if self.provider == "hf":
+            self.provider = "huggingface"
         self.codex_executable = None
         if self.provider == "codex":
             self.codex_executable = shutil.which("codex")
@@ -189,7 +246,10 @@ class GPTclient:
                     "`codex login` before using the Codex provider."
                 )
             self.client = None
-        elif self.provider == "azure" or api_version is not None:
+        elif self.provider == "azure" or (
+            self.provider not in _OPENAI_COMPATIBLE_PROVIDERS
+            and api_version is not None
+        ):
             self.client = AzureOpenAI(
                 azure_endpoint=endpoint,
                 api_key=api_key,
@@ -198,16 +258,23 @@ class GPTclient:
                 max_retries=0,
             )
         else:
-            self.client = OpenAI(
-                base_url=endpoint,
-                api_key=api_key,
-                timeout=timeout,
-                max_retries=0,
-            )
+            client_kwargs = {
+                "base_url": endpoint,
+                "api_key": api_key,
+                "timeout": timeout,
+                "max_retries": 0,
+            }
+            if self.provider == "huggingface" and bill_to:
+                client_kwargs["default_headers"] = {
+                    "X-HF-Bill-To": bill_to
+                }
+            self.client = OpenAI(**client_kwargs)
 
         self.endpoint = endpoint
         self.model_name = model_name
+        self.bill_to = bill_to if self.provider == "huggingface" else None
         self.timeout = timeout
+        self.last_usage = None
         self.image_formats = {".png", ".jpg", ".jpeg", ".webp", ".bmp", ".gif"}
         self.verbose = verbose
         if check_connection:
@@ -262,6 +329,116 @@ class GPTclient:
             ) from exc
         return target
 
+    def _prepare_huggingface_image(self, image: str | Image.Image) -> str:
+        """Encode one image as a compact data URL for Hugging Face."""
+        if isinstance(image, str) and image.startswith("data:"):
+            return image
+
+        if isinstance(image, Image.Image):
+            prepared_image = image.copy()
+        elif isinstance(image, str):
+            source = Path(image).expanduser()
+            try:
+                source_is_file = source.is_file()
+            except OSError:
+                source_is_file = False
+            if not source_is_file:
+                if source.suffix.lower() in self.image_formats:
+                    raise FileNotFoundError(f"Image file not found: {image}")
+                return f"data:image/png;base64,{image}"
+            try:
+                with Image.open(source) as source_image:
+                    prepared_image = source_image.copy()
+            except OSError as exc:
+                raise ValueError(f"Invalid image file: {image}") from exc
+        else:
+            raise TypeError(
+                "Image input must be a path, base64 string, or PIL Image"
+            )
+
+        prepared_image.thumbnail(
+            (HF_MAX_IMAGE_DIMENSION, HF_MAX_IMAGE_DIMENSION),
+            Image.Resampling.LANCZOS,
+        )
+        has_alpha = prepared_image.mode in {"LA", "RGBA"} or (
+            prepared_image.mode == "P" and "transparency" in prepared_image.info
+        )
+        buffer = BytesIO()
+        if has_alpha:
+            prepared_image.convert("RGBA").save(
+                buffer,
+                format="PNG",
+                optimize=True,
+            )
+            mime_type = "image/png"
+        else:
+            prepared_image.convert("RGB").save(
+                buffer,
+                format="JPEG",
+                quality=HF_IMAGE_JPEG_QUALITY,
+                optimize=True,
+            )
+            mime_type = "image/jpeg"
+        encoded = base64.b64encode(buffer.getvalue()).decode("utf-8")
+        return f"data:{mime_type};base64,{encoded}"
+
+    def _prepare_openai_image(self, image: str | Image.Image) -> str:
+        """Encode visible pixels as PNG for OpenAI-compatible APIs."""
+        if isinstance(image, Image.Image):
+            prepared_image = image.copy()
+        elif isinstance(image, str):
+            source = Path(image).expanduser()
+            try:
+                source_is_file = source.is_file()
+            except OSError:
+                source_is_file = False
+            if source_is_file:
+                try:
+                    with Image.open(source) as source_image:
+                        prepared_image = source_image.copy()
+                except OSError as exc:
+                    raise ValueError(f"Invalid image file: {image}") from exc
+            else:
+                if source.suffix.lower() in self.image_formats:
+                    raise FileNotFoundError(f"Image file not found: {image}")
+                encoded = image
+                if image.startswith("data:"):
+                    header, separator, encoded = image.partition(",")
+                    if not separator or ";base64" not in header.lower():
+                        raise ValueError(
+                            "Image data URI must contain base64 data"
+                        )
+                try:
+                    image_data = base64.b64decode(encoded, validate=True)
+                    with Image.open(BytesIO(image_data)) as decoded_image:
+                        prepared_image = decoded_image.copy()
+                except (OSError, ValueError) as exc:
+                    raise ValueError(
+                        "Image input is neither an existing image nor valid "
+                        "base64"
+                    ) from exc
+        else:
+            raise TypeError(
+                "Image input must be a path, base64 string, or PIL Image"
+            )
+
+        prepared_image.thumbnail(
+            (OPENAI_MAX_IMAGE_DIMENSION, OPENAI_MAX_IMAGE_DIMENSION),
+            Image.Resampling.LANCZOS,
+        )
+        has_alpha = prepared_image.mode in {"LA", "RGBA"} or (
+            prepared_image.mode == "P" and "transparency" in prepared_image.info
+        )
+        if has_alpha:
+            foreground = prepared_image.convert("RGBA")
+            background = Image.new("RGBA", foreground.size, (0, 0, 0, 255))
+            prepared_image = Image.alpha_composite(background, foreground)
+
+        buffer = BytesIO()
+        prepared_image.convert("RGB").save(buffer, format="PNG", optimize=True)
+        encoded = base64.b64encode(buffer.getvalue()).decode("utf-8")
+        return f"data:image/png;base64,{encoded}"
+
     def _query_codex(
         self,
         text_prompt: str,
@@ -345,10 +522,31 @@ class GPTclient:
         name = (model_name or "").lower()
         return "gpt-5" in name or "gpt5" in name
 
+    @staticmethod
+    def _is_reasoning_vlm(model_name: str) -> bool:
+        name = (model_name or "").lower()
+        return any(
+            model in name
+            for model in ("glm-4.5v", "glm-5.3-flash", "kimi-k3")
+        )
+
+    @staticmethod
+    def _default_top_p(model_name: str) -> float:
+        name = (model_name or "").lower()
+        return 0.95 if "kimi-k3" in name else 0.1
+
+    @staticmethod
+    def _default_reasoning_effort(model_name: str) -> str | None:
+        name = (model_name or "").lower()
+        if "glm-5.3-flash" in name:
+            return "max"
+        return "low" if "kimi-k3" in name else None
+
     @retry(
-        retry=retry_if_not_exception_type(openai.BadRequestError),
+        retry=retry_if_exception(_is_retryable_api_error),
         wait=wait_random_exponential(min=1, max=10),
         stop=stop_after_attempt(5) | stop_after_delay(DEFAULT_GPT_TIMEOUT),
+        reraise=True,
     )
     def completion_with_backoff(self, **kwargs):
         """Performs a chat completion request with retry/backoff."""
@@ -409,25 +607,22 @@ class GPTclient:
             if "openrouter" in self.endpoint:
                 image_base64 = combine_images_to_grid(image_base64)
             for img in image_base64:
-                if isinstance(img, Image.Image):
-                    buffer = BytesIO()
-                    img.save(buffer, format=img.format or "PNG")
-                    buffer.seek(0)
-                    image_binary = buffer.read()
-                    img = base64.b64encode(image_binary).decode("utf-8")
-                elif (
-                    len(os.path.splitext(img)) > 1
-                    and os.path.splitext(img)[-1].lower() in self.image_formats
-                ):
-                    if not os.path.exists(img):
-                        raise FileNotFoundError(f"Image file not found: {img}")
-                    with open(img, "rb") as f:
-                        img = base64.b64encode(f.read()).decode("utf-8")
-
+                if self.provider == "huggingface":
+                    content_user.append(
+                        {
+                            "type": "image_url",
+                            "image_url": {
+                                "url": self._prepare_huggingface_image(img)
+                            },
+                        }
+                    )
+                    continue
                 content_user.append(
                     {
                         "type": "image_url",
-                        "image_url": {"url": f"data:image/png;base64,{img}"},
+                        "image_url": {
+                            "url": self._prepare_openai_image(img)
+                        },
                     }
                 )
 
@@ -444,19 +639,27 @@ class GPTclient:
                 "model": self.model_name,
             }
         else:
+            max_tokens = (
+                REASONING_VLM_DEFAULT_MAX_TOKENS
+                if self._is_reasoning_vlm(self.model_name)
+                else 500
+            )
             payload = {
                 "messages": [
                     {"role": "system", "content": system_role},
                     {"role": "user", "content": content_user},
                 ],
                 "temperature": 0.1,
-                "max_tokens": 500,
-                "top_p": 0.1,
+                "max_tokens": max_tokens,
+                "top_p": self._default_top_p(self.model_name),
                 "frequency_penalty": 0,
                 "presence_penalty": 0,
                 "stop": None,
                 "model": self.model_name,
             }
+            reasoning_effort = self._default_reasoning_effort(self.model_name)
+            if reasoning_effort is not None:
+                payload["reasoning_effort"] = reasoning_effort
 
         if params:
             params = dict(params)
@@ -480,11 +683,19 @@ class GPTclient:
             payload.update(params)
 
         response = None
+        self.last_usage = None
         try:
-            response = self.completion_with_backoff(**payload)
-            response = response.choices[0].message.content
+            completion = self.completion_with_backoff(**payload)
+            usage = getattr(completion, "usage", None)
+            if usage is not None:
+                self.last_usage = (
+                    usage.model_dump()
+                    if hasattr(usage, "model_dump")
+                    else dict(usage)
+                )
+            response = completion.choices[0].message.content
         except Exception as e:
-            logger.error(f"Error GPTclint {self.endpoint} API call: {e}")
+            logger.error(f"Error GPTclient {self.endpoint} API call: {e}")
             response = None
 
         if self.verbose:
@@ -549,6 +760,7 @@ GPT_CLIENT = GPTclient(
     check_connection=False,
     timeout=DEFAULT_GPT_TIMEOUT,
     provider=settings["provider"],
+    bill_to=settings["bill_to"],
 )
 
 

embodied_gen/utils/gpt_config.yaml

@@ -1,27 +1,9 @@
 # config.yaml
-agent_type: "gpt-5.4" # gpt-4o, gpt-5.4 or gemma-4-31b or codex
+agent_type: "glm-5.3-flash"
 
-gpt-4o:
-  endpoint: https://xxx.openai.azure.com
-  api_key: xxx
-  api_version: 2025-xx-xx
-  model_name: yfb-gpt-4o
-
-gpt-5.4:
-  endpoint: https://yfb-openai-sweden.openai.azure.com/
-  api_key: xxx
-  api_version: 2024-12-01-preview
-  model_name: gpt-5.4
-
-gemma-4-31b:
-  endpoint: https://openrouter.ai/api/v1
-  api_key: sk-or-v1-xxx
-  api_version: null
-  model_name: google/gemma-4-31b-it:free
-
-codex:
-  provider: codex
-  endpoint: null
+glm-5.3-flash:
+  provider: huggingface
+  endpoint: https://router.huggingface.co/v1
   api_key: null
   api_version: null
-  model_name: null
+  model_name: zai-org/GLM-5.3-Flash:baseten

embodied_gen/validators/quality_checkers.py

@@ -155,6 +155,9 @@ class MeshGeoChecker(BaseChecker):
             self.prompt = """
             You are an expert in evaluating the geometry quality of generated 3D asset.
             You will be given rendered views of a generated 3D asset, type {}, with black background.
+            The input may be a contact sheet whose panels are different camera views of the same asset.
+            Do not treat separate view panels as duplicate object instances. Only report duplicate geometry
+            when duplication or overlap appears within an individual view.
             Your task is to evaluate the quality of the 3D asset generation,
             including geometry, structure, and appearance, based on the rendered views.
             Criteria: