From ab3f1a1cf17a9e07fa6584ca24cb6c6921d62ede Mon Sep 17 00:00:00 2001 From: ZenAlexa Date: Wed, 7 Oct 2026 08:43:16 -0700 Subject: [PATCH 1/2] [perf](kt-kernel): share checkpoint indexes across pending MoE layers --- kt-kernel/python/utils/amx.py | 94 +++++++++++--------------- kt-kernel/python/utils/loader.py | 112 ++++++++++++------------------- 2 files changed, 85 insertions(+), 121 deletions(-) diff --git a/kt-kernel/python/utils/amx.py b/kt-kernel/python/utils/amx.py index eaa7354ea..f715e5055 100644 --- a/kt-kernel/python/utils/amx.py +++ b/kt-kernel/python/utils/amx.py @@ -1,13 +1,10 @@ -import gc import glob -import logging import os +import weakref import torch import ctypes from typing import List, Optional -logger = logging.getLogger(__name__) - # Use relative imports for package structure from ..experts_base import BaseMoEWrapper from .loader import ( @@ -584,7 +581,8 @@ def load_weights(self, physical_to_logical_map_cpu: torch.Tensor): class NativeMoEWrapper(BaseMoEWrapper): """Wrapper for native CPU/SYCL experts stored in compressed SafeTensor format.""" - _native_loader_instance = None + # Pending layer wrappers own the index; the cache must not extend its lifetime. + _native_loader_cache = None def __init__( self, @@ -663,9 +661,7 @@ def __init__( "CPUINFER_USE_SYCL=1 using a SYCL compiler such as icpx." ) if method == "NVFP4" and not _HAS_AVX2_MXFP4_SUPPORT: - raise RuntimeError( - "NVFP4 needs the AVX2 FP4 backend (AVX2MXFP4_MOE), which is not compiled in." - ) + raise RuntimeError("NVFP4 needs the AVX2 FP4 backend (AVX2MXFP4_MOE), which is not compiled in.") if method == "MXFP4" and not (_HAS_MXFP4_SUPPORT or _HAS_AVX2_MXFP4_SUPPORT): raise RuntimeError( "MXFP4 backend not available. Required ISA (any one of):\n" @@ -681,6 +677,7 @@ def __init__( "Please recompile kt_kernel_ext with one of the above enabled." ) + weight_path = os.path.abspath(weight_path) super().__init__( layer_idx=layer_idx, num_experts=num_experts, @@ -699,9 +696,13 @@ def __init__( swiglu_limit=swiglu_limit, ) - if NativeMoEWrapper._native_loader_instance is None: - NativeMoEWrapper._native_loader_instance = NativeMoEWrapper._create_loader(method, weight_path) - self.loader = NativeMoEWrapper._native_loader_instance + loader_key = (method, weight_path) + cached_loader = NativeMoEWrapper._native_loader_cache + loader = cached_loader[1]() if cached_loader is not None and cached_loader[0] == loader_key else None + if loader is None: + loader = NativeMoEWrapper._create_loader(method, weight_path) + NativeMoEWrapper._native_loader_cache = (loader_key, weakref.ref(loader)) + self.loader = loader self.gate_weights = None self.up_weights = None @@ -731,22 +732,15 @@ def _create_loader(method: str, weight_path: str): else: raise NotImplementedError(f"Unsupported method for NativeMoEWrapper: {method}") - @staticmethod - def _release_loader(layer_idx: int = -1): - if NativeMoEWrapper._native_loader_instance is not None: - NativeMoEWrapper._native_loader_instance.close_all_handles() - NativeMoEWrapper._native_loader_instance = None - if layer_idx >= 0: - logger.info( - "[KT] Released NativeMoEWrapper loader after layer %d: " "safetensors mmap handles freed.", - layer_idx, - ) - else: - logger.info("[KT] Released NativeMoEWrapper loader: safetensors mmap handles freed.") - @staticmethod def force_release_loader(): - NativeMoEWrapper._release_loader() + """Close cached shard handles; pending layers retain their checkpoint index.""" + cached_loader = NativeMoEWrapper._native_loader_cache + NativeMoEWrapper._native_loader_cache = None + if cached_loader is not None: + loader = cached_loader[1]() + if loader is not None: + loader.close_all_handles() def load_weights_from_tensors( self, @@ -760,18 +754,9 @@ def load_weights_from_tensors( def load_weights(self, physical_to_logical_map_cpu: torch.Tensor): import time - if NativeMoEWrapper._native_loader_instance is None: - t_recreate_start = time.time() - NativeMoEWrapper._native_loader_instance = NativeMoEWrapper._create_loader(self.method, self.weight_path) - self.loader = NativeMoEWrapper._native_loader_instance - t_recreate_elapsed = (time.time() - t_recreate_start) * 1000 - logger.info( - "[KT] Recreated NativeMoEWrapper loader for layer %d (took %.1fms)", - self.layer_idx, - t_recreate_elapsed, - ) - else: - self.loader = NativeMoEWrapper._native_loader_instance + loader = self.loader + if loader is None: + loader = NativeMoEWrapper._create_loader(self.method, self.weight_path) t0 = time.time() _candidates = [ @@ -779,15 +764,23 @@ def load_weights(self, physical_to_logical_map_cpu: torch.Tensor): f"language_model.model.layers.{self.layer_idx}", f"model.language_model.layers.{self.layer_idx}", ] - weights = None - for base_key in _candidates: - try: - weights = self.loader.load_experts(base_key) - break - except (ValueError, KeyError): - continue - if weights is None: - raise ValueError(f"No experts found for layer {self.layer_idx} under any prefix: {_candidates}") + try: + weights = None + for base_key in _candidates: + try: + weights = loader.load_experts(base_key) + break + except (ValueError, KeyError): + continue + if weights is None: + raise ValueError(f"No experts found for layer {self.layer_idx} under any prefix: {_candidates}") + finally: + # Returned tensors own their storage through the native load and sync below. + self.loader = None + cached_loader = NativeMoEWrapper._native_loader_cache + if cached_loader is not None and cached_loader[1]() is loader: + NativeMoEWrapper._native_loader_cache = None + loader.close_all_handles() t1 = time.time() # Keep individual tensors instead of stacking - avoid expensive memory copy @@ -1027,7 +1020,6 @@ def load_weights(self, physical_to_logical_map_cpu: torch.Tensor): del self.up_scales del self.down_scales - NativeMoEWrapper._release_loader(layer_idx=self.layer_idx) t6 = time.time() print( @@ -1097,15 +1089,11 @@ def run_layerwise_fp8_batch( each rank's local H2D copy of expert ``e``. """ if self.method != "FP8": - raise RuntimeError( - "run_layerwise_fp8_batch is only valid for the block-FP8 NativeMoEWrapper backend" - ) + raise RuntimeError("run_layerwise_fp8_batch is only valid for the block-FP8 NativeMoEWrapper backend") if self.moe is None: raise RuntimeError("MoE instance not initialized; cannot run FP8 layerwise transport.") if not hasattr(self.moe, "run_layerwise_fp8_batch"): - raise NotImplementedError( - "The installed kt-kernel extension does not expose run_layerwise_fp8_batch." - ) + raise NotImplementedError("The installed kt-kernel extension does not expose run_layerwise_fp8_batch.") # The native batch calls the shared NUMA distributor directly. Drain # CPUInfer first so it cannot race a previously queued task against the # non-reentrant distributor state. diff --git a/kt-kernel/python/utils/loader.py b/kt-kernel/python/utils/loader.py index a4999f506..6266d969c 100644 --- a/kt-kernel/python/utils/loader.py +++ b/kt-kernel/python/utils/loader.py @@ -151,6 +151,7 @@ def __init__(self, file_path: str): self.__load_tensor_file_map(file_path) def __load_tensor_file_map(self, file_path: str): + file_path = os.path.abspath(file_path) if not os.path.exists(file_path): raise FileNotFoundError(f"Path not found: {file_path}") if os.path.isfile(file_path): @@ -169,20 +170,20 @@ def __load_tensor_file_map(self, file_path: str): if file.endswith(".safetensors"): found_safetensor = True file_path = os.path.join(root, file) - if file not in self.file_handle_map: + if file_path not in self.file_handle_map: try: handle = safe_open(file_path, framework="pt") - self.file_handle_map[file] = handle + self.file_handle_map[file_path] = handle except Exception as e: print(f"Error opening Safetensor file {file_path}: {e}") continue - f = self.file_handle_map.get(file) + f = self.file_handle_map.get(file_path) if f is None: continue try: for key in f.keys(): - self.tensor_file_map[key] = file + self.tensor_file_map[key] = file_path except Exception as e: print(f"Error reading Safetensor file {file_path}: {e}") @@ -192,23 +193,24 @@ def __load_tensor_file_map(self, file_path: str): def load_tensor(self, key: str, device: str = "cpu"): if key not in self.tensor_file_map: raise KeyError(f"Key {key} not found in Safetensor files") - file = self.tensor_file_map[key] - f = self.file_handle_map.get(file) + file_path = self.tensor_file_map[key] + f = self.file_handle_map.get(file_path) if f is None: - raise FileNotFoundError(f"File {file} not found in Safetensor files") + f = safe_open(file_path, framework="pt") + self.file_handle_map[file_path] = f tensor = f.get_tensor(key) + if device == "cpu": + return tensor return tensor.to(device) def close_all_handles(self): - """Close all file handles and clear the handle map. + """Release open shard handles while retaining the tensor index. - Note: safetensors.safe_open doesn't expose a close() method. Releasing - the mmap relies on reference counting: once file_handle_map is cleared - and no tensor holds a reference to the underlying mmap region, the OS - will reclaim the page cache. gc.collect() is called here to trigger - immediate reclamation rather than waiting for the next GC cycle. + Tensor views keep their mapped storage alive until the last view is released. + Subsequent loads reopen the required shards on demand. """ import gc + self.file_handle_map.clear() gc.collect() @@ -464,18 +466,6 @@ def _get_proj_names(self): _, gate, up, down = self.MOE_FORMATS[self._detected_format] return gate, up, down - def load_tensor(self, key: str, device: str = "cpu"): - if key not in self.tensor_file_map: - raise KeyError(f"Key {key} not found in Safetensor files") - file = self.tensor_file_map[key] - f = self.file_handle_map.get(file) - if f is None: - raise FileNotFoundError(f"File {file} not found in Safetensor files") - tensor = f.get_tensor(key) - if device == "cpu": - return tensor - return tensor.to(device) - def load_experts(self, base_key: str, device: str = "cpu"): """Load FP8 expert weights and their scale tensors. @@ -616,18 +606,6 @@ def _get_proj_names(self): _, gate, up, down = self.MOE_FORMATS[self._detected_format] return gate, up, down - def load_tensor(self, key: str, device: str = "cpu"): - if key not in self.tensor_file_map: - raise KeyError(f"Key {key} not found in Safetensor files") - file = self.tensor_file_map[key] - f = self.file_handle_map.get(file) - if f is None: - raise FileNotFoundError(f"File {file} not found in Safetensor files") - tensor = f.get_tensor(key) - if device == "cpu": - return tensor - return tensor.to(device) - def load_experts(self, base_key: str, device: str = "cpu"): """Load BF16 expert weights (no scales needed).""" if self._detected_format == "packed": @@ -1144,8 +1122,10 @@ def _verify_gptq_config(self, file_path): raise NotImplementedError( "GPTQ sym=false (asymmetric) is not supported. Only sym=true models are supported." ) - print(f"[GPTQSafeTensorLoader] Verified: sym={qc.get('sym')}, desc_act={qc.get('desc_act')}, " - f"bits={qc.get('bits')}, group_size={qc.get('group_size')}") + print( + f"[GPTQSafeTensorLoader] Verified: sym={qc.get('sym')}, desc_act={qc.get('desc_act')}, " + f"bits={qc.get('bits')}, group_size={qc.get('group_size')}" + ) def load_experts(self, base_key: str, device: str = "cpu"): """Load GPTQ expert qweight and scales. @@ -1176,13 +1156,23 @@ def load_experts(self, base_key: str, device: str = "cpu"): down_scales = [None] * expert_count for exp_id in range(expert_count): - gate_weights[exp_id] = self.load_tensor(f"{experts_prefix}.{exp_id}.{gate_name}.qweight", device).contiguous() + gate_weights[exp_id] = self.load_tensor( + f"{experts_prefix}.{exp_id}.{gate_name}.qweight", device + ).contiguous() up_weights[exp_id] = self.load_tensor(f"{experts_prefix}.{exp_id}.{up_name}.qweight", device).contiguous() - down_weights[exp_id] = self.load_tensor(f"{experts_prefix}.{exp_id}.{down_name}.qweight", device).contiguous() + down_weights[exp_id] = self.load_tensor( + f"{experts_prefix}.{exp_id}.{down_name}.qweight", device + ).contiguous() - gate_scales[exp_id] = self.load_tensor(f"{experts_prefix}.{exp_id}.{gate_name}.scales", device).float().contiguous() - up_scales[exp_id] = self.load_tensor(f"{experts_prefix}.{exp_id}.{up_name}.scales", device).float().contiguous() - down_scales[exp_id] = self.load_tensor(f"{experts_prefix}.{exp_id}.{down_name}.scales", device).float().contiguous() + gate_scales[exp_id] = ( + self.load_tensor(f"{experts_prefix}.{exp_id}.{gate_name}.scales", device).float().contiguous() + ) + up_scales[exp_id] = ( + self.load_tensor(f"{experts_prefix}.{exp_id}.{up_name}.scales", device).float().contiguous() + ) + down_scales[exp_id] = ( + self.load_tensor(f"{experts_prefix}.{exp_id}.{down_name}.scales", device).float().contiguous() + ) print(f"[GPTQSafeTensorLoader] Loaded {expert_count} experts from {experts_prefix}") return { @@ -1220,25 +1210,19 @@ def _experts_prefix_candidates(self, base_key: str) -> list[str]: return list(dict.fromkeys(candidates)) @staticmethod - def _ue8m0_to_bf16( - scale_t: torch.Tensor, *, reject_non_finite: bool = False - ) -> torch.Tensor: + def _ue8m0_to_bf16(scale_t: torch.Tensor, *, reject_non_finite: bool = False) -> torch.Tensor: if reject_non_finite: valid_dtypes = {torch.uint8} native_ue8m0 = getattr(torch, "float8_e8m0fnu", None) if native_ue8m0 is not None: valid_dtypes.add(native_ue8m0) if scale_t.dtype not in valid_dtypes: - raise TypeError( - "MXFP4 SFT requires raw uint8/float8_e8m0fnu scales, " - f"got {scale_t.dtype}" - ) + raise TypeError("MXFP4 SFT requires raw uint8/float8_e8m0fnu scales, " f"got {scale_t.dtype}") if scale_t.dtype != torch.uint8: scale_t = scale_t.view(torch.uint8) if reject_non_finite and torch.any(scale_t == 0xFF).item(): raise ValueError( - "MXFP4 SFT rejects reserved UE8M0 scale 0xff because it " - "would decode to a non-finite BF16 value" + "MXFP4 SFT rejects reserved UE8M0 scale 0xff because it " "would decode to a non-finite BF16 value" ) # BF16 normal exponent bits reproduce UE8M0 codes 1..254 exactly. # UE8M0 code 0 is 2^-127, represented by BF16 subnormal bits 0x0040. @@ -1266,9 +1250,7 @@ def load_experts( prefix = cand break if prefix is None: - raise ValueError( - f"No MXFP4 experts found under any of: {self._experts_prefix_candidates(base_key)}" - ) + raise ValueError(f"No MXFP4 experts found under any of: {self._experts_prefix_candidates(base_key)}") gate_weights = [None] * expert_count up_weights = [None] * expert_count @@ -1294,9 +1276,7 @@ def load_experts( (down_name, down_scales), ): s = self.load_tensor(f"{prefix}.{exp_id}.{proj}.scale", device) - dst[exp_id] = self._ue8m0_to_bf16( - s, reject_non_finite=reject_non_finite_scales - ) + dst[exp_id] = self._ue8m0_to_bf16(s, reject_non_finite=reject_non_finite_scales) print(f"[MXFP4SafeTensorLoader] Loaded {expert_count} experts from {prefix}") return { @@ -1338,13 +1318,13 @@ def _experts_prefix_candidates(self, base_key: str) -> list[str]: candidates = [self.EXPERTS_PATH_TPL.format(base=base_key)] for strip in ("model.language_model.", "language_model.model.", "language_model.", "model."): if base_key.startswith(strip): - candidates.append(self.EXPERTS_PATH_TPL.format(base=base_key[len(strip):])) + candidates.append(self.EXPERTS_PATH_TPL.format(base=base_key[len(strip) :])) # ModelOpt checkpoints for multimodal Qwen3.5-MoE nest under # model.language_model.layers.{L}; callers may pass either form. if not base_key.startswith("model.language_model."): for pre in ("model.language_model.", "language_model.model."): if base_key.startswith("model."): - candidates.append(self.EXPERTS_PATH_TPL.format(base=pre + base_key[len("model."):])) + candidates.append(self.EXPERTS_PATH_TPL.format(base=pre + base_key[len("model.") :])) return list(dict.fromkeys(candidates)) @staticmethod @@ -1365,9 +1345,7 @@ def load_experts(self, base_key: str, device: str = "cpu"): prefix = cand break if prefix is None: - raise ValueError( - f"No NVFP4 experts found under any of: {self._experts_prefix_candidates(base_key)}" - ) + raise ValueError(f"No NVFP4 experts found under any of: {self._experts_prefix_candidates(base_key)}") gate_weights = [None] * expert_count up_weights = [None] * expert_count @@ -1423,7 +1401,7 @@ def _experts_prefix_candidates(self, base_key: str) -> list[str]: candidates = [self.EXPERTS_PATH_TPL.format(base=base_key)] for strip in ("language_model.model.", "language_model.", "model."): if base_key.startswith(strip): - candidates.append(self.EXPERTS_PATH_TPL.format(base=base_key[len(strip):])) + candidates.append(self.EXPERTS_PATH_TPL.format(base=base_key[len(strip) :])) return list(dict.fromkeys(candidates)) def load_experts(self, base_key: str, device: str = "cpu"): @@ -1438,9 +1416,7 @@ def load_experts(self, base_key: str, device: str = "cpu"): prefix = cand break if prefix is None: - raise ValueError( - f"No MXFP8 experts found under any of: {self._experts_prefix_candidates(base_key)}" - ) + raise ValueError(f"No MXFP8 experts found under any of: {self._experts_prefix_candidates(base_key)}") gate_weights = [None] * expert_count up_weights = [None] * expert_count From cc1d127d4d8adcd7cada7c0e63f8f2559f3d58a7 Mon Sep 17 00:00:00 2001 From: ZenAlexa Date: Wed, 7 Oct 2026 09:24:08 -0700 Subject: [PATCH 2/2] [perf](kt-kernel): release shard handles without full-heap collection --- kt-kernel/python/utils/loader.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/kt-kernel/python/utils/loader.py b/kt-kernel/python/utils/loader.py index 6266d969c..345b218dc 100644 --- a/kt-kernel/python/utils/loader.py +++ b/kt-kernel/python/utils/loader.py @@ -209,10 +209,7 @@ def close_all_handles(self): Tensor views keep their mapped storage alive until the last view is released. Subsequent loads reopen the required shards on demand. """ - import gc - self.file_handle_map.clear() - gc.collect() def load_experts(self, base_key: str, device: str = "cpu"): """