Skip to content
Merged
82 changes: 38 additions & 44 deletions kt-kernel/python/utils/amx.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,10 @@
import gc
import glob
import logging
import os
import weakref
import torch
import ctypes
from typing import List, Optional

logger = logging.getLogger(__name__)

# Use relative imports for package structure
from ..experts_base import BaseMoEWrapper
from .loader import (
Expand Down Expand Up @@ -608,7 +605,8 @@ def load_weights(self, physical_to_logical_map_cpu: torch.Tensor):
class NativeMoEWrapper(BaseMoEWrapper):
"""Wrapper for native CPU/SYCL experts stored in compressed SafeTensor format."""

_native_loader_instance = None
# Pending layer wrappers own the index; the cache must not extend its lifetime.
_native_loader_cache = None

def __init__(
self,
Expand Down Expand Up @@ -703,6 +701,7 @@ def __init__(
"Please recompile kt_kernel_ext with one of the above enabled."
)

weight_path = os.path.abspath(weight_path)
super().__init__(
layer_idx=layer_idx,
num_experts=num_experts,
Expand All @@ -721,9 +720,13 @@ def __init__(
swiglu_limit=swiglu_limit,
)

if NativeMoEWrapper._native_loader_instance is None:
NativeMoEWrapper._native_loader_instance = NativeMoEWrapper._create_loader(method, weight_path)
self.loader = NativeMoEWrapper._native_loader_instance
loader_key = (method, weight_path)
cached_loader = NativeMoEWrapper._native_loader_cache
loader = cached_loader[1]() if cached_loader is not None and cached_loader[0] == loader_key else None
if loader is None:
loader = NativeMoEWrapper._create_loader(method, weight_path)
NativeMoEWrapper._native_loader_cache = (loader_key, weakref.ref(loader))
self.loader = loader

self.gate_weights = None
self.up_weights = None
Expand Down Expand Up @@ -753,22 +756,15 @@ def _create_loader(method: str, weight_path: str):
else:
raise NotImplementedError(f"Unsupported method for NativeMoEWrapper: {method}")

@staticmethod
def _release_loader(layer_idx: int = -1):
if NativeMoEWrapper._native_loader_instance is not None:
NativeMoEWrapper._native_loader_instance.close_all_handles()
NativeMoEWrapper._native_loader_instance = None
if layer_idx >= 0:
logger.info(
"[KT] Released NativeMoEWrapper loader after layer %d: " "safetensors mmap handles freed.",
layer_idx,
)
else:
logger.info("[KT] Released NativeMoEWrapper loader: safetensors mmap handles freed.")

@staticmethod
def force_release_loader():
NativeMoEWrapper._release_loader()
"""Close cached shard handles; pending layers retain their checkpoint index."""
cached_loader = NativeMoEWrapper._native_loader_cache
NativeMoEWrapper._native_loader_cache = None
if cached_loader is not None:
loader = cached_loader[1]()
if loader is not None:
loader.close_all_handles()

def load_weights_from_tensors(
self,
Expand All @@ -782,34 +778,33 @@ def load_weights_from_tensors(
def load_weights(self, physical_to_logical_map_cpu: torch.Tensor):
import time

if NativeMoEWrapper._native_loader_instance is None:
t_recreate_start = time.time()
NativeMoEWrapper._native_loader_instance = NativeMoEWrapper._create_loader(self.method, self.weight_path)
self.loader = NativeMoEWrapper._native_loader_instance
t_recreate_elapsed = (time.time() - t_recreate_start) * 1000
logger.info(
"[KT] Recreated NativeMoEWrapper loader for layer %d (took %.1fms)",
self.layer_idx,
t_recreate_elapsed,
)
else:
self.loader = NativeMoEWrapper._native_loader_instance
loader = self.loader
if loader is None:
loader = NativeMoEWrapper._create_loader(self.method, self.weight_path)

t0 = time.time()
_candidates = [
f"model.layers.{self.layer_idx}",
f"language_model.model.layers.{self.layer_idx}",
f"model.language_model.layers.{self.layer_idx}",
]
weights = None
for base_key in _candidates:
try:
weights = self.loader.load_experts(base_key)
break
except (ValueError, KeyError):
continue
if weights is None:
raise ValueError(f"No experts found for layer {self.layer_idx} under any prefix: {_candidates}")
try:
weights = None
for base_key in _candidates:
try:
weights = loader.load_experts(base_key)
break
except (ValueError, KeyError):
continue
if weights is None:
raise ValueError(f"No experts found for layer {self.layer_idx} under any prefix: {_candidates}")
finally:
# Returned tensors own their storage through the native load and sync below.
self.loader = None
cached_loader = NativeMoEWrapper._native_loader_cache
if cached_loader is not None and cached_loader[1]() is loader:
NativeMoEWrapper._native_loader_cache = None
loader.close_all_handles()
t1 = time.time()

# Keep individual tensors instead of stacking - avoid expensive memory copy
Expand Down Expand Up @@ -1054,7 +1049,6 @@ def load_weights(self, physical_to_logical_map_cpu: torch.Tensor):
del self.up_scales
del self.down_scales

NativeMoEWrapper._release_loader(layer_idx=self.layer_idx)
t6 = time.time()

print(
Expand Down
113 changes: 43 additions & 70 deletions kt-kernel/python/utils/loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ def __init__(self, file_path: str):
self.__load_tensor_file_map(file_path)

def __load_tensor_file_map(self, file_path: str):
file_path = os.path.abspath(file_path)
if not os.path.exists(file_path):
raise FileNotFoundError(f"Path not found: {file_path}")
if os.path.isfile(file_path):
Expand All @@ -169,20 +170,20 @@ def __load_tensor_file_map(self, file_path: str):
if file.endswith(".safetensors"):
found_safetensor = True
file_path = os.path.join(root, file)
if file not in self.file_handle_map:
if file_path not in self.file_handle_map:
try:
handle = safe_open(file_path, framework="pt")
self.file_handle_map[file] = handle
self.file_handle_map[file_path] = handle
except Exception as e:
print(f"Error opening Safetensor file {file_path}: {e}")
continue

f = self.file_handle_map.get(file)
f = self.file_handle_map.get(file_path)
if f is None:
continue
try:
for key in f.keys():
self.tensor_file_map[key] = file
self.tensor_file_map[key] = file_path
except Exception as e:
print(f"Error reading Safetensor file {file_path}: {e}")

Expand All @@ -192,25 +193,23 @@ def __load_tensor_file_map(self, file_path: str):
def load_tensor(self, key: str, device: str = "cpu"):
if key not in self.tensor_file_map:
raise KeyError(f"Key {key} not found in Safetensor files")
file = self.tensor_file_map[key]
f = self.file_handle_map.get(file)
file_path = self.tensor_file_map[key]
f = self.file_handle_map.get(file_path)
if f is None:
raise FileNotFoundError(f"File {file} not found in Safetensor files")
f = safe_open(file_path, framework="pt")
self.file_handle_map[file_path] = f
tensor = f.get_tensor(key)
if device == "cpu":
return tensor
return tensor.to(device)

def close_all_handles(self):
"""Close all file handles and clear the handle map.
"""Release open shard handles while retaining the tensor index.

Note: safetensors.safe_open doesn't expose a close() method. Releasing
the mmap relies on reference counting: once file_handle_map is cleared
and no tensor holds a reference to the underlying mmap region, the OS
will reclaim the page cache. gc.collect() is called here to trigger
immediate reclamation rather than waiting for the next GC cycle.
Tensor views keep their mapped storage alive until the last view is released.
Subsequent loads reopen the required shards on demand.
"""
import gc
self.file_handle_map.clear()
gc.collect()

def load_experts(self, base_key: str, device: str = "cpu"):
"""
Expand Down Expand Up @@ -464,18 +463,6 @@ def _get_proj_names(self):
_, gate, up, down = self.MOE_FORMATS[self._detected_format]
return gate, up, down

def load_tensor(self, key: str, device: str = "cpu"):
if key not in self.tensor_file_map:
raise KeyError(f"Key {key} not found in Safetensor files")
file = self.tensor_file_map[key]
f = self.file_handle_map.get(file)
if f is None:
raise FileNotFoundError(f"File {file} not found in Safetensor files")
tensor = f.get_tensor(key)
if device == "cpu":
return tensor
return tensor.to(device)

def load_experts(self, base_key: str, device: str = "cpu"):
"""Load FP8 expert weights and their scale tensors.

Expand Down Expand Up @@ -616,18 +603,6 @@ def _get_proj_names(self):
_, gate, up, down = self.MOE_FORMATS[self._detected_format]
return gate, up, down

def load_tensor(self, key: str, device: str = "cpu"):
if key not in self.tensor_file_map:
raise KeyError(f"Key {key} not found in Safetensor files")
file = self.tensor_file_map[key]
f = self.file_handle_map.get(file)
if f is None:
raise FileNotFoundError(f"File {file} not found in Safetensor files")
tensor = f.get_tensor(key)
if device == "cpu":
return tensor
return tensor.to(device)

def load_experts(self, base_key: str, device: str = "cpu"):
"""Load BF16 expert weights (no scales needed)."""
if self._detected_format == "packed":
Expand Down Expand Up @@ -1144,8 +1119,10 @@ def _verify_gptq_config(self, file_path):
raise NotImplementedError(
"GPTQ sym=false (asymmetric) is not supported. Only sym=true models are supported."
)
print(f"[GPTQSafeTensorLoader] Verified: sym={qc.get('sym')}, desc_act={qc.get('desc_act')}, "
f"bits={qc.get('bits')}, group_size={qc.get('group_size')}")
print(
f"[GPTQSafeTensorLoader] Verified: sym={qc.get('sym')}, desc_act={qc.get('desc_act')}, "
f"bits={qc.get('bits')}, group_size={qc.get('group_size')}"
)

def load_experts(self, base_key: str, device: str = "cpu"):
"""Load GPTQ expert qweight and scales.
Expand Down Expand Up @@ -1176,13 +1153,23 @@ def load_experts(self, base_key: str, device: str = "cpu"):
down_scales = [None] * expert_count

for exp_id in range(expert_count):
gate_weights[exp_id] = self.load_tensor(f"{experts_prefix}.{exp_id}.{gate_name}.qweight", device).contiguous()
gate_weights[exp_id] = self.load_tensor(
f"{experts_prefix}.{exp_id}.{gate_name}.qweight", device
).contiguous()
up_weights[exp_id] = self.load_tensor(f"{experts_prefix}.{exp_id}.{up_name}.qweight", device).contiguous()
down_weights[exp_id] = self.load_tensor(f"{experts_prefix}.{exp_id}.{down_name}.qweight", device).contiguous()
down_weights[exp_id] = self.load_tensor(
f"{experts_prefix}.{exp_id}.{down_name}.qweight", device
).contiguous()

gate_scales[exp_id] = self.load_tensor(f"{experts_prefix}.{exp_id}.{gate_name}.scales", device).float().contiguous()
up_scales[exp_id] = self.load_tensor(f"{experts_prefix}.{exp_id}.{up_name}.scales", device).float().contiguous()
down_scales[exp_id] = self.load_tensor(f"{experts_prefix}.{exp_id}.{down_name}.scales", device).float().contiguous()
gate_scales[exp_id] = (
self.load_tensor(f"{experts_prefix}.{exp_id}.{gate_name}.scales", device).float().contiguous()
)
up_scales[exp_id] = (
self.load_tensor(f"{experts_prefix}.{exp_id}.{up_name}.scales", device).float().contiguous()
)
down_scales[exp_id] = (
self.load_tensor(f"{experts_prefix}.{exp_id}.{down_name}.scales", device).float().contiguous()
)

print(f"[GPTQSafeTensorLoader] Loaded {expert_count} experts from {experts_prefix}")
return {
Expand Down Expand Up @@ -1220,25 +1207,19 @@ def _experts_prefix_candidates(self, base_key: str) -> list[str]:
return list(dict.fromkeys(candidates))

@staticmethod
def _ue8m0_to_bf16(
scale_t: torch.Tensor, *, reject_non_finite: bool = False
) -> torch.Tensor:
def _ue8m0_to_bf16(scale_t: torch.Tensor, *, reject_non_finite: bool = False) -> torch.Tensor:
if reject_non_finite:
valid_dtypes = {torch.uint8}
native_ue8m0 = getattr(torch, "float8_e8m0fnu", None)
if native_ue8m0 is not None:
valid_dtypes.add(native_ue8m0)
if scale_t.dtype not in valid_dtypes:
raise TypeError(
"MXFP4 SFT requires raw uint8/float8_e8m0fnu scales, "
f"got {scale_t.dtype}"
)
raise TypeError("MXFP4 SFT requires raw uint8/float8_e8m0fnu scales, " f"got {scale_t.dtype}")
if scale_t.dtype != torch.uint8:
scale_t = scale_t.view(torch.uint8)
if reject_non_finite and torch.any(scale_t == 0xFF).item():
raise ValueError(
"MXFP4 SFT rejects reserved UE8M0 scale 0xff because it "
"would decode to a non-finite BF16 value"
"MXFP4 SFT rejects reserved UE8M0 scale 0xff because it " "would decode to a non-finite BF16 value"
)
# BF16 normal exponent bits reproduce UE8M0 codes 1..254 exactly.
# UE8M0 code 0 is 2^-127, represented by BF16 subnormal bits 0x0040.
Expand Down Expand Up @@ -1266,9 +1247,7 @@ def load_experts(
prefix = cand
break
if prefix is None:
raise ValueError(
f"No MXFP4 experts found under any of: {self._experts_prefix_candidates(base_key)}"
)
raise ValueError(f"No MXFP4 experts found under any of: {self._experts_prefix_candidates(base_key)}")

gate_weights = [None] * expert_count
up_weights = [None] * expert_count
Expand All @@ -1294,9 +1273,7 @@ def load_experts(
(down_name, down_scales),
):
s = self.load_tensor(f"{prefix}.{exp_id}.{proj}.scale", device)
dst[exp_id] = self._ue8m0_to_bf16(
s, reject_non_finite=reject_non_finite_scales
)
dst[exp_id] = self._ue8m0_to_bf16(s, reject_non_finite=reject_non_finite_scales)

print(f"[MXFP4SafeTensorLoader] Loaded {expert_count} experts from {prefix}")
return {
Expand Down Expand Up @@ -1338,13 +1315,13 @@ def _experts_prefix_candidates(self, base_key: str) -> list[str]:
candidates = [self.EXPERTS_PATH_TPL.format(base=base_key)]
for strip in ("model.language_model.", "language_model.model.", "language_model.", "model."):
if base_key.startswith(strip):
candidates.append(self.EXPERTS_PATH_TPL.format(base=base_key[len(strip):]))
candidates.append(self.EXPERTS_PATH_TPL.format(base=base_key[len(strip) :]))
# ModelOpt checkpoints for multimodal Qwen3.5-MoE nest under
# model.language_model.layers.{L}; callers may pass either form.
if not base_key.startswith("model.language_model."):
for pre in ("model.language_model.", "language_model.model."):
if base_key.startswith("model."):
candidates.append(self.EXPERTS_PATH_TPL.format(base=pre + base_key[len("model."):]))
candidates.append(self.EXPERTS_PATH_TPL.format(base=pre + base_key[len("model.") :]))
return list(dict.fromkeys(candidates))

@staticmethod
Expand All @@ -1365,9 +1342,7 @@ def load_experts(self, base_key: str, device: str = "cpu"):
prefix = cand
break
if prefix is None:
raise ValueError(
f"No NVFP4 experts found under any of: {self._experts_prefix_candidates(base_key)}"
)
raise ValueError(f"No NVFP4 experts found under any of: {self._experts_prefix_candidates(base_key)}")

gate_weights = [None] * expert_count
up_weights = [None] * expert_count
Expand Down Expand Up @@ -1423,7 +1398,7 @@ def _experts_prefix_candidates(self, base_key: str) -> list[str]:
candidates = [self.EXPERTS_PATH_TPL.format(base=base_key)]
for strip in ("language_model.model.", "language_model.", "model."):
if base_key.startswith(strip):
candidates.append(self.EXPERTS_PATH_TPL.format(base=base_key[len(strip):]))
candidates.append(self.EXPERTS_PATH_TPL.format(base=base_key[len(strip) :]))
return list(dict.fromkeys(candidates))

def load_experts(self, base_key: str, device: str = "cpu"):
Expand All @@ -1438,9 +1413,7 @@ def load_experts(self, base_key: str, device: str = "cpu"):
prefix = cand
break
if prefix is None:
raise ValueError(
f"No MXFP8 experts found under any of: {self._experts_prefix_candidates(base_key)}"
)
raise ValueError(f"No MXFP8 experts found under any of: {self._experts_prefix_candidates(base_key)}")

gate_weights = [None] * expert_count
up_weights = [None] * expert_count
Expand Down