Alexander Claude Opus 5.5 (1M context) commited on
Commit
6f89b98
路
1 Parent(s): a4eb9c1

Give up on a conditioner that never answers

Browse files

Since 2026-10-01 every encode logged `[gpu] encode` and then nothing: no
booking, no error, no GPU spent, while the same /encode called from
outside the Space finished in 12s. `predict` waits forever, so wait a
bounded time instead, log where the job stood and whether a token was
forwarded, cancel it, and drop the shared client in case its stream went
stale.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

Files changed (1) hide show
  1. h3_engine.py +27 -1
h3_engine.py CHANGED
@@ -16,6 +16,7 @@ import tempfile
16
  import time
17
  import traceback
18
  from collections import OrderedDict
 
19
  from functools import cache
20
 
21
  # `import spaces` patches `torch.cuda` before anything can initialize it, which is what lets the 77 GB load happen at
@@ -32,6 +33,9 @@ import lora_stack
32
 
33
  MODEL_REPO = os.getenv("H3_MODEL_REPO", "MiniMaxAI/MiniMax-H3")
34
  CONDITIONER_SPACE = os.getenv("H3_CONDITIONER", "multimodalart/qwen3vl-conditioner")
 
 
 
35
  GPU_SIZE = os.getenv("H3_GPU_SIZE", "xlarge")
36
  # cuDNN's fused attention needs nothing installed and beats the SDPA default on this pool.
37
  ATTENTION = os.getenv("H3_ATTENTION", "_native_cudnn")
@@ -719,7 +723,11 @@ def encode(prompt, first, last, canvas, num_frames, ip_token=None):
719
  print("[h3] conditioning served from cache", flush=True)
720
  return _encode_cache[key]
721
 
722
- path, plan = _conditioner(ip_token).predict(
 
 
 
 
723
  prompt=prompt,
724
  image_path=handle_file(first) if first else None,
725
  last_image_path=handle_file(last) if last else None,
@@ -728,6 +736,24 @@ def encode(prompt, first, last, canvas, num_frames, ip_token=None):
728
  rewrite_prompt=False,
729
  api_name="/encode",
730
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
731
  with safe_open(path, framework="pt") as handle:
732
  conditioning = (handle.get_tensor("prompt_embeds"), handle.get_tensor("text_token_tags"), handle.metadata(), plan)
733
 
 
16
  import time
17
  import traceback
18
  from collections import OrderedDict
19
+ from concurrent.futures import TimeoutError as FutureTimeout
20
  from functools import cache
21
 
22
  # `import spaces` patches `torch.cuda` before anything can initialize it, which is what lets the 77 GB load happen at
 
33
 
34
  MODEL_REPO = os.getenv("H3_MODEL_REPO", "MiniMaxAI/MiniMax-H3")
35
  CONDITIONER_SPACE = os.getenv("H3_CONDITIONER", "multimodalart/qwen3vl-conditioner")
36
+ # An encode takes about 40s with a cold ZeroGPU slot and 12s warm; this is room for a queue on the conditioner's
37
+ # side, not for a job nobody is running.
38
+ CONDITIONER_TIMEOUT = int(os.getenv("H3_CONDITIONER_TIMEOUT", "180"))
39
  GPU_SIZE = os.getenv("H3_GPU_SIZE", "xlarge")
40
  # cuDNN's fused attention needs nothing installed and beats the SDPA default on this pool.
41
  ATTENTION = os.getenv("H3_ATTENTION", "_native_cudnn")
 
723
  print("[h3] conditioning served from cache", flush=True)
724
  return _encode_cache[key]
725
 
726
+ # `predict` has no timeout, and a job the conditioner never answers held the request forever: from 2026-10-01
727
+ # every encode logged `[gpu] encode` and then nothing - no booking, no error, no GPU spent - while the same call
728
+ # from outside finished in 12s. So wait a bounded time, say where the job stood, and let the request fail.
729
+ client = _conditioner(ip_token)
730
+ job = client.submit(
731
  prompt=prompt,
732
  image_path=handle_file(first) if first else None,
733
  last_image_path=handle_file(last) if last else None,
 
736
  rewrite_prompt=False,
737
  api_name="/encode",
738
  )
739
+ try:
740
+ path, plan = job.result(timeout=CONDITIONER_TIMEOUT)
741
+ except FutureTimeout:
742
+ status = job.status()
743
+ print(
744
+ f"[h3] conditioner silent for {CONDITIONER_TIMEOUT}s 路 job {status.code.name}"
745
+ f" 路 rank {status.rank} of {status.queue_size} 路 token {'forwarded' if ip_token else 'none'}",
746
+ flush=True,
747
+ )
748
+ job.cancel()
749
+ # The shared client is held for the life of the process; if its stream to the conditioner went stale,
750
+ # keeping it would hang every request that has no token of its own.
751
+ if not ip_token:
752
+ _shared_conditioner.cache_clear()
753
+ raise TimeoutError(
754
+ f"the conditioner `{CONDITIONER_SPACE}` did not answer in {CONDITIONER_TIMEOUT}s "
755
+ f"(job {status.code.name.lower()}) - no GPU time was booked, try again"
756
+ ) from None
757
  with safe_open(path, framework="pt") as handle:
758
  conditioning = (handle.get_tensor("prompt_embeds"), handle.get_tensor("text_token_tags"), handle.metadata(), plan)
759