RDS777 commited on
Commit
9b87155
·
1 Parent(s): e61ebbe

v22.0 — Qwen2.5-Coder-32B-Instruct, max hardware L4 24GB

Browse files
Files changed (2) hide show
  1. download_model.py +3 -3
  2. main.py +20 -7
download_model.py CHANGED
@@ -1,8 +1,8 @@
1
  from huggingface_hub import snapshot_download
2
- print("[DOWNLOAD] Qwen4-Coder-32B-A3B...")
3
  snapshot_download(
4
- "Qwen/Qwen4-Coder-32B-A3B",
5
- local_dir="/app/models/Qwen4-Coder",
6
  local_dir_use_symlinks=False,
7
  )
8
  print("[DOWNLOAD] Done!")
 
1
  from huggingface_hub import snapshot_download
2
+ print("[DOWNLOAD] Qwen2.5-Coder-32B-Instruct (~20GB)...")
3
  snapshot_download(
4
+ "Qwen/Qwen2.5-Coder-32B-Instruct",
5
+ local_dir="/app/models/Qwen2.5-Coder-32B",
6
  local_dir_use_symlinks=False,
7
  )
8
  print("[DOWNLOAD] Done!")
main.py CHANGED
@@ -22,7 +22,7 @@ from sympy.parsing.sympy_parser import parse_expr
22
  logging.basicConfig(level=logging.INFO)
23
  logger = logging.getLogger("atypik")
24
  VERSION = "22.0"
25
- MODEL_NAME = os.getenv("MODEL_NAME", "Qwen/Qwen4-Coder-32B-A3B")
26
  # =============================================================================
27
  # COGNITIVE ENGINE
28
  # =============================================================================
@@ -41,18 +41,29 @@ class CognitiveEngine:
41
  self._tokenizer = AutoTokenizer.from_pretrained(
42
  self.model_name, trust_remote_code=True, use_fast=False
43
  )
 
44
  self._model = AutoModelForCausalLM.from_pretrained(
45
  self.model_name,
46
  device_map="auto",
47
  load_in_4bit=True,
48
- bnb_4bit_compute_dtype=torch.float16,
49
- trust_remote_code=True
 
 
50
  )
51
- logger.info("[ENGINE] Loaded on %s", "cuda" if torch.cuda.is_available() else "cpu")
 
 
 
 
52
  except Exception as e:
53
  logger.error("[ENGINE] Failed: %s", e)
54
- if "Qwen3" in self.model_name:
55
- logger.warning("[ENGINE] Fallback to Qwen2")
 
 
 
 
56
  self.model_name = "Qwen/Qwen2-7B-Instruct"
57
  self._load()
58
  else:
@@ -71,7 +82,7 @@ class CognitiveEngine:
71
  loop = asyncio.get_event_loop()
72
  await loop.run_in_executor(None, self._load)
73
  await loop.run_in_executor(None, self._load_embedder)
74
- def generate(self, prompt, max_tokens=512, temperature=0.7):
75
  if not self._loaded:
76
  self._load()
77
  messages = [
@@ -524,3 +535,5 @@ if __name__ == "__main__":
524
 
525
 
526
 
 
 
 
22
  logging.basicConfig(level=logging.INFO)
23
  logger = logging.getLogger("atypik")
24
  VERSION = "22.0"
25
+ MODEL_NAME = os.getenv("MODEL_NAME", "Qwen/Qwen2.5-Coder-32B-Instruct")
26
  # =============================================================================
27
  # COGNITIVE ENGINE
28
  # =============================================================================
 
41
  self._tokenizer = AutoTokenizer.from_pretrained(
42
  self.model_name, trust_remote_code=True, use_fast=False
43
  )
44
+ # 32B sur 24GB : 4-bit + bfloat16 + gradient checkpointing disabled
45
  self._model = AutoModelForCausalLM.from_pretrained(
46
  self.model_name,
47
  device_map="auto",
48
  load_in_4bit=True,
49
+ bnb_4bit_compute_dtype=torch.bfloat16,
50
+ bnb_4bit_use_double_quant=True, # Double quantization = moins de VRAM
51
+ trust_remote_code=True,
52
+ torch_dtype=torch.bfloat16,
53
  )
54
+ # Désactiver le cache KV pour économiser VRAM (sacrifice latence)
55
+ self._model.config.use_cache = False
56
+ logger.info("[ENGINE] Loaded on %s | VRAM used: ~%.1f GB",
57
+ "cuda" if torch.cuda.is_available() else "cpu",
58
+ torch.cuda.memory_allocated() / 1e9 if torch.cuda.is_available() else 0)
59
  except Exception as e:
60
  logger.error("[ENGINE] Failed: %s", e)
61
+ if "Qwen2.5-Coder-32B" in self.model_name:
62
+ logger.warning("[ENGINE] Fallback to Qwen2.5-Coder-14B")
63
+ self.model_name = "Qwen/Qwen2.5-Coder-14B-Instruct"
64
+ self._load()
65
+ elif "Qwen2.5-Coder-14B" in self.model_name:
66
+ logger.warning("[ENGINE] Fallback to Qwen2-7B")
67
  self.model_name = "Qwen/Qwen2-7B-Instruct"
68
  self._load()
69
  else:
 
82
  loop = asyncio.get_event_loop()
83
  await loop.run_in_executor(None, self._load)
84
  await loop.run_in_executor(None, self._load_embedder)
85
+ def generate(self, prompt, max_tokens=1024, temperature=0.7):
86
  if not self._loaded:
87
  self._load()
88
  messages = [
 
535
 
536
 
537
 
538
+
539
+