JonasGeiping commited on
Commit
8811223
·
verified ·
1 Parent(s): 2617dc6

Update stream_inference.py

Browse files
Files changed (1) hide show
  1. stream_inference.py +0 -3
stream_inference.py CHANGED
@@ -194,9 +194,6 @@ def generate(
194
  # for j in range(N):
195
  # if (j // C) < (i // C) or j == i:
196
  # mask[0,0,i,j] = 0.0
197
- # but for typical prefill sizes (n_prefill = 100-300 rows → N = 1000-3000
198
- # tokens), the loop is 1-9 million iterations and takes 20-60s — long
199
- # enough that users assume the Seed button is broken. Vectorized below.
200
  rows_idx = torch.arange(N, device=device) // C
201
  allowed = (rows_idx.unsqueeze(0) < rows_idx.unsqueeze(1)) | torch.eye(
202
  N, dtype=torch.bool, device=device
 
194
  # for j in range(N):
195
  # if (j // C) < (i // C) or j == i:
196
  # mask[0,0,i,j] = 0.0
 
 
 
197
  rows_idx = torch.arange(N, device=device) // C
198
  allowed = (rows_idx.unsqueeze(0) < rows_idx.unsqueeze(1)) | torch.eye(
199
  N, dtype=torch.bool, device=device