v22.0 — Qwen2.5-Coder-32B-Instruct, max hardware L4 24GB
Browse files- download_model.py +3 -3
- main.py +20 -7
download_model.py
CHANGED
|
@@ -1,8 +1,8 @@
|
|
| 1 |
from huggingface_hub import snapshot_download
|
| 2 |
-
print("[DOWNLOAD]
|
| 3 |
snapshot_download(
|
| 4 |
-
"Qwen/
|
| 5 |
-
local_dir="/app/models/
|
| 6 |
local_dir_use_symlinks=False,
|
| 7 |
)
|
| 8 |
print("[DOWNLOAD] Done!")
|
|
|
|
| 1 |
from huggingface_hub import snapshot_download
|
| 2 |
+
print("[DOWNLOAD] Qwen2.5-Coder-32B-Instruct (~20GB)...")
|
| 3 |
snapshot_download(
|
| 4 |
+
"Qwen/Qwen2.5-Coder-32B-Instruct",
|
| 5 |
+
local_dir="/app/models/Qwen2.5-Coder-32B",
|
| 6 |
local_dir_use_symlinks=False,
|
| 7 |
)
|
| 8 |
print("[DOWNLOAD] Done!")
|
main.py
CHANGED
|
@@ -22,7 +22,7 @@ from sympy.parsing.sympy_parser import parse_expr
|
|
| 22 |
logging.basicConfig(level=logging.INFO)
|
| 23 |
logger = logging.getLogger("atypik")
|
| 24 |
VERSION = "22.0"
|
| 25 |
-
MODEL_NAME = os.getenv("MODEL_NAME", "Qwen/
|
| 26 |
# =============================================================================
|
| 27 |
# COGNITIVE ENGINE
|
| 28 |
# =============================================================================
|
|
@@ -41,18 +41,29 @@ class CognitiveEngine:
|
|
| 41 |
self._tokenizer = AutoTokenizer.from_pretrained(
|
| 42 |
self.model_name, trust_remote_code=True, use_fast=False
|
| 43 |
)
|
|
|
|
| 44 |
self._model = AutoModelForCausalLM.from_pretrained(
|
| 45 |
self.model_name,
|
| 46 |
device_map="auto",
|
| 47 |
load_in_4bit=True,
|
| 48 |
-
bnb_4bit_compute_dtype=torch.
|
| 49 |
-
|
|
|
|
|
|
|
| 50 |
)
|
| 51 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
except Exception as e:
|
| 53 |
logger.error("[ENGINE] Failed: %s", e)
|
| 54 |
-
if "
|
| 55 |
-
logger.warning("[ENGINE] Fallback to Qwen2")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
self.model_name = "Qwen/Qwen2-7B-Instruct"
|
| 57 |
self._load()
|
| 58 |
else:
|
|
@@ -71,7 +82,7 @@ class CognitiveEngine:
|
|
| 71 |
loop = asyncio.get_event_loop()
|
| 72 |
await loop.run_in_executor(None, self._load)
|
| 73 |
await loop.run_in_executor(None, self._load_embedder)
|
| 74 |
-
def generate(self, prompt, max_tokens=
|
| 75 |
if not self._loaded:
|
| 76 |
self._load()
|
| 77 |
messages = [
|
|
@@ -524,3 +535,5 @@ if __name__ == "__main__":
|
|
| 524 |
|
| 525 |
|
| 526 |
|
|
|
|
|
|
|
|
|
| 22 |
logging.basicConfig(level=logging.INFO)
|
| 23 |
logger = logging.getLogger("atypik")
|
| 24 |
VERSION = "22.0"
|
| 25 |
+
MODEL_NAME = os.getenv("MODEL_NAME", "Qwen/Qwen2.5-Coder-32B-Instruct")
|
| 26 |
# =============================================================================
|
| 27 |
# COGNITIVE ENGINE
|
| 28 |
# =============================================================================
|
|
|
|
| 41 |
self._tokenizer = AutoTokenizer.from_pretrained(
|
| 42 |
self.model_name, trust_remote_code=True, use_fast=False
|
| 43 |
)
|
| 44 |
+
# 32B sur 24GB : 4-bit + bfloat16 + gradient checkpointing disabled
|
| 45 |
self._model = AutoModelForCausalLM.from_pretrained(
|
| 46 |
self.model_name,
|
| 47 |
device_map="auto",
|
| 48 |
load_in_4bit=True,
|
| 49 |
+
bnb_4bit_compute_dtype=torch.bfloat16,
|
| 50 |
+
bnb_4bit_use_double_quant=True, # Double quantization = moins de VRAM
|
| 51 |
+
trust_remote_code=True,
|
| 52 |
+
torch_dtype=torch.bfloat16,
|
| 53 |
)
|
| 54 |
+
# Désactiver le cache KV pour économiser VRAM (sacrifice latence)
|
| 55 |
+
self._model.config.use_cache = False
|
| 56 |
+
logger.info("[ENGINE] Loaded on %s | VRAM used: ~%.1f GB",
|
| 57 |
+
"cuda" if torch.cuda.is_available() else "cpu",
|
| 58 |
+
torch.cuda.memory_allocated() / 1e9 if torch.cuda.is_available() else 0)
|
| 59 |
except Exception as e:
|
| 60 |
logger.error("[ENGINE] Failed: %s", e)
|
| 61 |
+
if "Qwen2.5-Coder-32B" in self.model_name:
|
| 62 |
+
logger.warning("[ENGINE] Fallback to Qwen2.5-Coder-14B")
|
| 63 |
+
self.model_name = "Qwen/Qwen2.5-Coder-14B-Instruct"
|
| 64 |
+
self._load()
|
| 65 |
+
elif "Qwen2.5-Coder-14B" in self.model_name:
|
| 66 |
+
logger.warning("[ENGINE] Fallback to Qwen2-7B")
|
| 67 |
self.model_name = "Qwen/Qwen2-7B-Instruct"
|
| 68 |
self._load()
|
| 69 |
else:
|
|
|
|
| 82 |
loop = asyncio.get_event_loop()
|
| 83 |
await loop.run_in_executor(None, self._load)
|
| 84 |
await loop.run_in_executor(None, self._load_embedder)
|
| 85 |
+
def generate(self, prompt, max_tokens=1024, temperature=0.7):
|
| 86 |
if not self._loaded:
|
| 87 |
self._load()
|
| 88 |
messages = [
|
|
|
|
| 535 |
|
| 536 |
|
| 537 |
|
| 538 |
+
|
| 539 |
+
|