diff --git a/kt-kernel/python/utils/amx.py b/kt-kernel/python/utils/amx.py index 20a60e7af..1981b07d5 100644 --- a/kt-kernel/python/utils/amx.py +++ b/kt-kernel/python/utils/amx.py @@ -1,13 +1,10 @@ -import gc import glob -import logging import os +import weakref import torch import ctypes from typing import List, Optional -logger = logging.getLogger(__name__) - # Use relative imports for package structure from ..experts_base import BaseMoEWrapper from .loader import ( @@ -608,7 +605,8 @@ def load_weights(self, physical_to_logical_map_cpu: torch.Tensor): class NativeMoEWrapper(BaseMoEWrapper): """Wrapper for native CPU/SYCL experts stored in compressed SafeTensor format.""" - _native_loader_instance = None + # Pending layer wrappers own the index; the cache must not extend its lifetime. + _native_loader_cache = None def __init__( self, @@ -703,6 +701,7 @@ def __init__( "Please recompile kt_kernel_ext with one of the above enabled." ) + weight_path = os.path.abspath(weight_path) super().__init__( layer_idx=layer_idx, num_experts=num_experts, @@ -721,9 +720,13 @@ def __init__( swiglu_limit=swiglu_limit, ) - if NativeMoEWrapper._native_loader_instance is None: - NativeMoEWrapper._native_loader_instance = NativeMoEWrapper._create_loader(method, weight_path) - self.loader = NativeMoEWrapper._native_loader_instance + loader_key = (method, weight_path) + cached_loader = NativeMoEWrapper._native_loader_cache + loader = cached_loader[1]() if cached_loader is not None and cached_loader[0] == loader_key else None + if loader is None: + loader = NativeMoEWrapper._create_loader(method, weight_path) + NativeMoEWrapper._native_loader_cache = (loader_key, weakref.ref(loader)) + self.loader = loader self.gate_weights = None self.up_weights = None @@ -753,22 +756,15 @@ def _create_loader(method: str, weight_path: str): else: raise NotImplementedError(f"Unsupported method for NativeMoEWrapper: {method}") - @staticmethod - def _release_loader(layer_idx: int = -1): - if NativeMoEWrapper._native_loader_instance is not None: - NativeMoEWrapper._native_loader_instance.close_all_handles() - NativeMoEWrapper._native_loader_instance = None - if layer_idx >= 0: - logger.info( - "[KT] Released NativeMoEWrapper loader after layer %d: " "safetensors mmap handles freed.", - layer_idx, - ) - else: - logger.info("[KT] Released NativeMoEWrapper loader: safetensors mmap handles freed.") - @staticmethod def force_release_loader(): - NativeMoEWrapper._release_loader() + """Close cached shard handles; pending layers retain their checkpoint index.""" + cached_loader = NativeMoEWrapper._native_loader_cache + NativeMoEWrapper._native_loader_cache = None + if cached_loader is not None: + loader = cached_loader[1]() + if loader is not None: + loader.close_all_handles() def load_weights_from_tensors( self, @@ -782,18 +778,9 @@ def load_weights_from_tensors( def load_weights(self, physical_to_logical_map_cpu: torch.Tensor): import time - if NativeMoEWrapper._native_loader_instance is None: - t_recreate_start = time.time() - NativeMoEWrapper._native_loader_instance = NativeMoEWrapper._create_loader(self.method, self.weight_path) - self.loader = NativeMoEWrapper._native_loader_instance - t_recreate_elapsed = (time.time() - t_recreate_start) * 1000 - logger.info( - "[KT] Recreated NativeMoEWrapper loader for layer %d (took %.1fms)", - self.layer_idx, - t_recreate_elapsed, - ) - else: - self.loader = NativeMoEWrapper._native_loader_instance + loader = self.loader + if loader is None: + loader = NativeMoEWrapper._create_loader(self.method, self.weight_path) t0 = time.time() _candidates = [ @@ -801,15 +788,23 @@ def load_weights(self, physical_to_logical_map_cpu: torch.Tensor): f"language_model.model.layers.{self.layer_idx}", f"model.language_model.layers.{self.layer_idx}", ] - weights = None - for base_key in _candidates: - try: - weights = self.loader.load_experts(base_key) - break - except (ValueError, KeyError): - continue - if weights is None: - raise ValueError(f"No experts found for layer {self.layer_idx} under any prefix: {_candidates}") + try: + weights = None + for base_key in _candidates: + try: + weights = loader.load_experts(base_key) + break + except (ValueError, KeyError): + continue + if weights is None: + raise ValueError(f"No experts found for layer {self.layer_idx} under any prefix: {_candidates}") + finally: + # Returned tensors own their storage through the native load and sync below. + self.loader = None + cached_loader = NativeMoEWrapper._native_loader_cache + if cached_loader is not None and cached_loader[1]() is loader: + NativeMoEWrapper._native_loader_cache = None + loader.close_all_handles() t1 = time.time() # Keep individual tensors instead of stacking - avoid expensive memory copy @@ -1054,7 +1049,6 @@ def load_weights(self, physical_to_logical_map_cpu: torch.Tensor): del self.up_scales del self.down_scales - NativeMoEWrapper._release_loader(layer_idx=self.layer_idx) t6 = time.time() print( diff --git a/kt-kernel/python/utils/loader.py b/kt-kernel/python/utils/loader.py index a4999f506..345b218dc 100644 --- a/kt-kernel/python/utils/loader.py +++ b/kt-kernel/python/utils/loader.py @@ -151,6 +151,7 @@ def __init__(self, file_path: str): self.__load_tensor_file_map(file_path) def __load_tensor_file_map(self, file_path: str): + file_path = os.path.abspath(file_path) if not os.path.exists(file_path): raise FileNotFoundError(f"Path not found: {file_path}") if os.path.isfile(file_path): @@ -169,20 +170,20 @@ def __load_tensor_file_map(self, file_path: str): if file.endswith(".safetensors"): found_safetensor = True file_path = os.path.join(root, file) - if file not in self.file_handle_map: + if file_path not in self.file_handle_map: try: handle = safe_open(file_path, framework="pt") - self.file_handle_map[file] = handle + self.file_handle_map[file_path] = handle except Exception as e: print(f"Error opening Safetensor file {file_path}: {e}") continue - f = self.file_handle_map.get(file) + f = self.file_handle_map.get(file_path) if f is None: continue try: for key in f.keys(): - self.tensor_file_map[key] = file + self.tensor_file_map[key] = file_path except Exception as e: print(f"Error reading Safetensor file {file_path}: {e}") @@ -192,25 +193,23 @@ def __load_tensor_file_map(self, file_path: str): def load_tensor(self, key: str, device: str = "cpu"): if key not in self.tensor_file_map: raise KeyError(f"Key {key} not found in Safetensor files") - file = self.tensor_file_map[key] - f = self.file_handle_map.get(file) + file_path = self.tensor_file_map[key] + f = self.file_handle_map.get(file_path) if f is None: - raise FileNotFoundError(f"File {file} not found in Safetensor files") + f = safe_open(file_path, framework="pt") + self.file_handle_map[file_path] = f tensor = f.get_tensor(key) + if device == "cpu": + return tensor return tensor.to(device) def close_all_handles(self): - """Close all file handles and clear the handle map. + """Release open shard handles while retaining the tensor index. - Note: safetensors.safe_open doesn't expose a close() method. Releasing - the mmap relies on reference counting: once file_handle_map is cleared - and no tensor holds a reference to the underlying mmap region, the OS - will reclaim the page cache. gc.collect() is called here to trigger - immediate reclamation rather than waiting for the next GC cycle. + Tensor views keep their mapped storage alive until the last view is released. + Subsequent loads reopen the required shards on demand. """ - import gc self.file_handle_map.clear() - gc.collect() def load_experts(self, base_key: str, device: str = "cpu"): """ @@ -464,18 +463,6 @@ def _get_proj_names(self): _, gate, up, down = self.MOE_FORMATS[self._detected_format] return gate, up, down - def load_tensor(self, key: str, device: str = "cpu"): - if key not in self.tensor_file_map: - raise KeyError(f"Key {key} not found in Safetensor files") - file = self.tensor_file_map[key] - f = self.file_handle_map.get(file) - if f is None: - raise FileNotFoundError(f"File {file} not found in Safetensor files") - tensor = f.get_tensor(key) - if device == "cpu": - return tensor - return tensor.to(device) - def load_experts(self, base_key: str, device: str = "cpu"): """Load FP8 expert weights and their scale tensors. @@ -616,18 +603,6 @@ def _get_proj_names(self): _, gate, up, down = self.MOE_FORMATS[self._detected_format] return gate, up, down - def load_tensor(self, key: str, device: str = "cpu"): - if key not in self.tensor_file_map: - raise KeyError(f"Key {key} not found in Safetensor files") - file = self.tensor_file_map[key] - f = self.file_handle_map.get(file) - if f is None: - raise FileNotFoundError(f"File {file} not found in Safetensor files") - tensor = f.get_tensor(key) - if device == "cpu": - return tensor - return tensor.to(device) - def load_experts(self, base_key: str, device: str = "cpu"): """Load BF16 expert weights (no scales needed).""" if self._detected_format == "packed": @@ -1144,8 +1119,10 @@ def _verify_gptq_config(self, file_path): raise NotImplementedError( "GPTQ sym=false (asymmetric) is not supported. Only sym=true models are supported." ) - print(f"[GPTQSafeTensorLoader] Verified: sym={qc.get('sym')}, desc_act={qc.get('desc_act')}, " - f"bits={qc.get('bits')}, group_size={qc.get('group_size')}") + print( + f"[GPTQSafeTensorLoader] Verified: sym={qc.get('sym')}, desc_act={qc.get('desc_act')}, " + f"bits={qc.get('bits')}, group_size={qc.get('group_size')}" + ) def load_experts(self, base_key: str, device: str = "cpu"): """Load GPTQ expert qweight and scales. @@ -1176,13 +1153,23 @@ def load_experts(self, base_key: str, device: str = "cpu"): down_scales = [None] * expert_count for exp_id in range(expert_count): - gate_weights[exp_id] = self.load_tensor(f"{experts_prefix}.{exp_id}.{gate_name}.qweight", device).contiguous() + gate_weights[exp_id] = self.load_tensor( + f"{experts_prefix}.{exp_id}.{gate_name}.qweight", device + ).contiguous() up_weights[exp_id] = self.load_tensor(f"{experts_prefix}.{exp_id}.{up_name}.qweight", device).contiguous() - down_weights[exp_id] = self.load_tensor(f"{experts_prefix}.{exp_id}.{down_name}.qweight", device).contiguous() + down_weights[exp_id] = self.load_tensor( + f"{experts_prefix}.{exp_id}.{down_name}.qweight", device + ).contiguous() - gate_scales[exp_id] = self.load_tensor(f"{experts_prefix}.{exp_id}.{gate_name}.scales", device).float().contiguous() - up_scales[exp_id] = self.load_tensor(f"{experts_prefix}.{exp_id}.{up_name}.scales", device).float().contiguous() - down_scales[exp_id] = self.load_tensor(f"{experts_prefix}.{exp_id}.{down_name}.scales", device).float().contiguous() + gate_scales[exp_id] = ( + self.load_tensor(f"{experts_prefix}.{exp_id}.{gate_name}.scales", device).float().contiguous() + ) + up_scales[exp_id] = ( + self.load_tensor(f"{experts_prefix}.{exp_id}.{up_name}.scales", device).float().contiguous() + ) + down_scales[exp_id] = ( + self.load_tensor(f"{experts_prefix}.{exp_id}.{down_name}.scales", device).float().contiguous() + ) print(f"[GPTQSafeTensorLoader] Loaded {expert_count} experts from {experts_prefix}") return { @@ -1220,25 +1207,19 @@ def _experts_prefix_candidates(self, base_key: str) -> list[str]: return list(dict.fromkeys(candidates)) @staticmethod - def _ue8m0_to_bf16( - scale_t: torch.Tensor, *, reject_non_finite: bool = False - ) -> torch.Tensor: + def _ue8m0_to_bf16(scale_t: torch.Tensor, *, reject_non_finite: bool = False) -> torch.Tensor: if reject_non_finite: valid_dtypes = {torch.uint8} native_ue8m0 = getattr(torch, "float8_e8m0fnu", None) if native_ue8m0 is not None: valid_dtypes.add(native_ue8m0) if scale_t.dtype not in valid_dtypes: - raise TypeError( - "MXFP4 SFT requires raw uint8/float8_e8m0fnu scales, " - f"got {scale_t.dtype}" - ) + raise TypeError("MXFP4 SFT requires raw uint8/float8_e8m0fnu scales, " f"got {scale_t.dtype}") if scale_t.dtype != torch.uint8: scale_t = scale_t.view(torch.uint8) if reject_non_finite and torch.any(scale_t == 0xFF).item(): raise ValueError( - "MXFP4 SFT rejects reserved UE8M0 scale 0xff because it " - "would decode to a non-finite BF16 value" + "MXFP4 SFT rejects reserved UE8M0 scale 0xff because it " "would decode to a non-finite BF16 value" ) # BF16 normal exponent bits reproduce UE8M0 codes 1..254 exactly. # UE8M0 code 0 is 2^-127, represented by BF16 subnormal bits 0x0040. @@ -1266,9 +1247,7 @@ def load_experts( prefix = cand break if prefix is None: - raise ValueError( - f"No MXFP4 experts found under any of: {self._experts_prefix_candidates(base_key)}" - ) + raise ValueError(f"No MXFP4 experts found under any of: {self._experts_prefix_candidates(base_key)}") gate_weights = [None] * expert_count up_weights = [None] * expert_count @@ -1294,9 +1273,7 @@ def load_experts( (down_name, down_scales), ): s = self.load_tensor(f"{prefix}.{exp_id}.{proj}.scale", device) - dst[exp_id] = self._ue8m0_to_bf16( - s, reject_non_finite=reject_non_finite_scales - ) + dst[exp_id] = self._ue8m0_to_bf16(s, reject_non_finite=reject_non_finite_scales) print(f"[MXFP4SafeTensorLoader] Loaded {expert_count} experts from {prefix}") return { @@ -1338,13 +1315,13 @@ def _experts_prefix_candidates(self, base_key: str) -> list[str]: candidates = [self.EXPERTS_PATH_TPL.format(base=base_key)] for strip in ("model.language_model.", "language_model.model.", "language_model.", "model."): if base_key.startswith(strip): - candidates.append(self.EXPERTS_PATH_TPL.format(base=base_key[len(strip):])) + candidates.append(self.EXPERTS_PATH_TPL.format(base=base_key[len(strip) :])) # ModelOpt checkpoints for multimodal Qwen3.5-MoE nest under # model.language_model.layers.{L}; callers may pass either form. if not base_key.startswith("model.language_model."): for pre in ("model.language_model.", "language_model.model."): if base_key.startswith("model."): - candidates.append(self.EXPERTS_PATH_TPL.format(base=pre + base_key[len("model."):])) + candidates.append(self.EXPERTS_PATH_TPL.format(base=pre + base_key[len("model.") :])) return list(dict.fromkeys(candidates)) @staticmethod @@ -1365,9 +1342,7 @@ def load_experts(self, base_key: str, device: str = "cpu"): prefix = cand break if prefix is None: - raise ValueError( - f"No NVFP4 experts found under any of: {self._experts_prefix_candidates(base_key)}" - ) + raise ValueError(f"No NVFP4 experts found under any of: {self._experts_prefix_candidates(base_key)}") gate_weights = [None] * expert_count up_weights = [None] * expert_count @@ -1423,7 +1398,7 @@ def _experts_prefix_candidates(self, base_key: str) -> list[str]: candidates = [self.EXPERTS_PATH_TPL.format(base=base_key)] for strip in ("language_model.model.", "language_model.", "model."): if base_key.startswith(strip): - candidates.append(self.EXPERTS_PATH_TPL.format(base=base_key[len(strip):])) + candidates.append(self.EXPERTS_PATH_TPL.format(base=base_key[len(strip) :])) return list(dict.fromkeys(candidates)) def load_experts(self, base_key: str, device: str = "cpu"): @@ -1438,9 +1413,7 @@ def load_experts(self, base_key: str, device: str = "cpu"): prefix = cand break if prefix is None: - raise ValueError( - f"No MXFP8 experts found under any of: {self._experts_prefix_candidates(base_key)}" - ) + raise ValueError(f"No MXFP8 experts found under any of: {self._experts_prefix_candidates(base_key)}") gate_weights = [None] * expert_count up_weights = [None] * expert_count