assets/model_adapter_template.py
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
"""
BaseModelAdaptationadapterTemplate
suitable for HuggingFace Transformers of Decoder-only LLM.
"""
from typing import List, Any, Generator
from torch import nn
from msmodelslim.core.base.protocol import ProcessRequest
from msmodelslim.core.const import DeviceType
from msmodelslim.model.common.layer_wise_forward import (
generated_decoder_layer_visit_func,
transformers_generated_forward_func
)
from msmodelslim.model.common.transformers import TransformersModel
from msmodelslim.model.interface_hub import (
ModelInfoInterface,
ModelSlimPipelineInterfaceV1
)
from msmodelslim.utils.logging import logger_setter
@logger_setter()
class MyModelAdapter(TransformersModel,
ModelInfoInterface,
ModelSlimPipelineInterfaceV1):
# ==================== ModelInfoInterface ====================
def get_model_pedigree(self) -> str:
return "my_model"
def get_model_type(self) -> str:
return self.model_type
# ==================== ModelSlimPipelineInterfaceV1 ====================
def handle_dataset(self, dataset: Any, device: DeviceType = DeviceType.NPU) -> List[Any]:
return self._get_tokenized_data(dataset, device)
def init_model(self, device: DeviceType = DeviceType.NPU) -> nn.Module:
return self._load_model(device)
def generate_model_visit(self, model: nn.Module) -> Generator[ProcessRequest, Any, None]:
yield from generated_decoder_layer_visit_func(model)
def generate_model_forward(self, model: nn.Module, inputs: Any) -> Generator[ProcessRequest, Any, None]:
yield from transformers_generated_forward_func(model, inputs)
def enable_kv_cache(self, model: nn.Module, need_kv_cache: bool) -> None:
return self._enable_kv_cache(model, need_kv_cache)
assets/vlm_model_adapter_template.py
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
"""
multiplemodelstatemanagesolveModel (VLM) AdaptationadapterTemplate.
"""
from pathlib import Path
from typing import Any, Generator, List
from torch import nn
from transformers import AutoProcessor
from msmodelslim.app.naive_quantization.model_info_interface import ModelInfoInterface
from msmodelslim.core.base.protocol import ProcessRequest
from msmodelslim.core.const import DeviceType
from msmodelslim.model.common.layer_wise_forward import generated_decoder_layer_visit_func, transformers_generated_forward_func
from msmodelslim.model.common.vlm_base import VLMBaseModelAdapter
from msmodelslim.model.interface_hub import ModelSlimPipelineInterfaceV1
from msmodelslim.utils.exception import InvalidModelError
from msmodelslim.utils.logging import logger_setter
from msmodelslim.utils.security import get_valid_read_path
@logger_setter()
class MyVLMModelAdapter(
VLMBaseModelAdapter,
ModelInfoInterface,
ModelSlimPipelineInterfaceV1,
):
"""VLM BaseTemplate. """
def __init__(self, model_type: str, model_path: Path, trust_remote_code: bool = False):
self._processor = None
super().__init__(model_type, model_path, trust_remote_code)
# ==================== ModelInfoInterface ====================
def get_model_pedigree(self) -> str:
return "my_vlm"
def get_model_type(self) -> str:
return self.model_type
# ==================== ModelSlimPipelineInterfaceV1 ====================
def handle_dataset(self, dataset: Any, device: DeviceType = DeviceType.NPU) -> List[Any]:
"""
willfiguredocumentsamplethisconvertexchangeascalibratestandardoutputinput.
samplethisSuggestformatformula: item.text + item.image (or dict: {"text": ..., "image": ...}) .
"""
self._processor = AutoProcessor.from_pretrained(
self.model_path,
trust_remote_code=self.trust_remote_code,
local_files_only=True,
)
processed = []
for item in dataset:
text = item.text if hasattr(item, "text") else item.get("text")
image = item.image if hasattr(item, "image") else item.get("image")
if text is None or image is None:
raise InvalidModelError(
"VLM calibratestandardsamplethisrequiressametimePackagecontain text and image. ",
action="pleaseProvides image+text Data, avoidavoidpuredocumentthissamplethis. ",
)
image = get_valid_read_path(str(image))
messages = [{
"role": "user",
"content": [
{"type": "image", "image": str(image)},
{"type": "text", "text": str(text)},
],
}]
inputs = self._processor.apply_chat_template(
messages,
tokenize=True,
add_generation_prompt=True,
return_dict=True,
return_tensors="pt",
)
processed.append(
self._collect_inputs_to_device(
inputs,
device,
keys=[
"input_ids",
"attention_mask",
"position_ids",
"pixel_values",
"pixel_values_videos",
"image_grid_thw",
"video_grid_thw",
"cache_position",
],
defaults={},
)
)
return processed
def init_model(self, device: DeviceType = DeviceType.NPU) -> nn.Module:
return self._load_model(device)
def generate_model_visit(self, model: nn.Module) -> Generator[ProcessRequest, Any, None]:
yield from generated_decoder_layer_visit_func(model)
def generate_model_forward(self, model: nn.Module, inputs: Any) -> Generator[ProcessRequest, Any, None]:
yield from transformers_generated_forward_func(model, inputs)
def enable_kv_cache(self, model: nn.Module, need_kv_cache: bool) -> None:
return self._enable_kv_cache(model, need_kv_cache)references/acceptance-criteria.md
# Acceptance Criteria
## Functional Acceptance Criteria
### 1. Model Analysis
| Criteria | Description | Verification Method |
|----------|-------------|---------------------|
| AC-1.1 | Should correctly identify model implementation source | Check transformers or model-local detection |
| AC-1.2 | Should identify model type (LLM/VLM/MoE) | Verify output matches expected type |
| AC-1.3 | Should detect layer-by-layer loading requirements | Check output analysis report |
### 2. Adapter Creation
| Criteria | Description | Verification Method |
|----------|-------------|---------------------|
| AC-2.1 | Should generate adapter code from template | Compare with expected template output |
| AC-2.2 | Should implement all 5 required interfaces | Run verification scripts |
| AC-2.3 | Should handle MoE unpack for fused weights | Verify unpack logic correctness |
### 3. Adapter Registration
| Criteria | Description | Verification Method |
|----------|-------------|---------------------|
| AC-3.1 | Should register model in config.ini | Check config file entries |
| AC-3.2 | Should execute install.sh successfully | Verify no errors during installation |
| AC-3.3 | Should import adapter module correctly | Test Python import |
### 4. Verification Workflow
| Criteria | Description | Verification Method |
|----------|-------------|---------------------|
| AC-4.1 | Step 1: Generate test model succeeds | Check step1 output |
| AC-4.2 | Step 2: Full fallback quantization passes | Check step2 output |
| AC-4.3 | Step 3: Weight verification matches float | Check tolerance within 1e-5 |
| AC-4.4 | Step 4: Quant description validation passes | Check step4 output |
## Correct/Error Pattern Comparison
### Adapter Interface Implementation
**Correct:** All 5 required interfaces implemented
```python
class MyModelAdapter(TransformersModel, ModelSlimPipelineInterfaceV1):
def handle_dataset(self, raw_data, device): ...
def init_model(self, config, device): ...
def generate_model_visit(self): ...
def generate_model_forward(self): ...
def enable_kv_cache(self, model): ...
```
**Error:** Missing required interfaces
```python
class MyModelAdapter:
def init_model(self, config, device): ...
# Missing: handle_dataset, generate_model_visit, generate_model_forward, enable_kv_cache
```
### MoE Weight Handling
**Correct:** Unpack fused weights before quantization
```python
# For 3D packed experts [num_experts, hidden, intermediate]
gate = weight[0::3] # Split by 3
up = weight[1::3]
down = weight[2::3]
```
**Error:** Quantize without unpacking
```python
quantize(weight) # Wrong: fused 3D weights
```
## Non-Functional Acceptance Criteria
| Criteria | Description | Threshold |
|----------|-------------|-----------|
| NAC-1.1 | Adapter code generation time | < 5 seconds |
| NAC-1.2 | Weight verification tolerance | <= 1e-5 |
| NAC-1.3 | Model loading compatibility | Python 3.8+ |
## Test Cases Summary
### Positive Test Cases
1. TC-001: Decoder-only LLM adapter creation
2. TC-002: VLM text backbone adapter creation
3. TC-003: MoE model adapter with unpacking
4. TC-004: Full 4-step verification workflow
5. TC-005: Adapter registration and import
### Negative Test Cases
1. TC-N01: Unsupported model type (multimodal generation)
2. TC-N02: Missing required interfaces
3. TC-N03: MoE weights without unpacking
4. TC-N04: Registration without install.sh
5. TC-N05: Weight verification tolerance exceeded
references/core_workflow.md
# Core Workflow (Create + Verification)
this Skill willBaseAdaptationAdapter CreationandBaseVerificationMergeasoneitemProcess.
## Phase A: CreateAdaptationadapter
1. selectselectTemplate:
- LLM Usage `model_adapter_template.py`
- VLM documentthisPathUsage `vlm_model_adapter_template.py`
2. ImplementationmustneedInterface.
3. in `config/config.ini` middleRegistrationModeltypetypeandEntry.
## Phase B: VerificationAdaptationadapter (mustneedfourstep)
Execute in Orderin order tounderCheck:
1. step1: Generate Random WeightsTestingModel
2. step2: allFallbackQuantization
3. step3: Verification Step2 allFallbackModeland Step1 floatpointModelWeightsstrictformatoneconsistent, andConfirmModelcancompleteadjustLoad/keepkeep
4. step4: VerificationactualactualQuantizationProcesspositiveoften (W8A8 quietstate/movestate) andthroughexceedDescriptionfileRulesCheck
## verifyreceiveRules
onlywhenPhase A andPhase B (Step1~Step4) averagethroughexceedtime, onlywillAdaptationadapterstandardrememberascompleted.
references/implementation_guide.md
# AdaptationadapterImplementationGuide
## Directory Structure (Suggest)
onecountModelAdaptationadapterDirectory Usually Contains at LeastPackagecontainin order tounderfile:
```text
msmodelslim/model/<model_type>/
├── __init__.py
├── model_adapter.py
└── model.py
```
- `__init__.py`: mustmustkeepin, keepcertifyDirectorycanas Python Packagebeguideinput
- `model_adapter.py`: AdaptationadapterEntryand 5 countmustneedInterfaceImplementation
- `model.py`: ModelStructureRelatedImplementation (ifdistributelayervisitask/previousdirectionassistaidlogiclogic)
ifresultoughtModelalreadyhaveotherotheraccordingdependfile (if `utils.py`, `configuration_*.py`) , according toactualactualrequiressupplementfill, butnotneedsavestrategy `__init__.py`.
## mustneedInterface
1. `handle_dataset`
2. `init_model`
3. `generate_model_visit`
4. `generate_model_forward`
5. `enable_kv_cache`
## according toTemplateregiondistribute: LLM / VLM mustneedInterface
in order tounderconclusiontheorybased on `assets/model_adapter_template.py` and `assets/vlm_model_adapter_template.py`.
### LLM (Decoder-only)
- **pushrecommendcontinueadmit**: `TransformersModel + ModelSlimPipelineInterfaceV1` (`ModelInfoInterface` canselectbutSuggest)
- **mustmustImplementation** (5 count) : `handle_dataset`, `init_model`, `generate_model_visit`, `generate_model_forward`, `enable_kv_cache`
- **TemplatemiddleoftenseeassistaidMethod** (nonFrameworkstrongmake, butmultiplenumberModelrequires) : `generate_decoder_layer`, `_decoder_layer_prefix`, `_load_decoder_if_not_exist`, `_create_model_instance`
### VLM (multiplemodelstatemanagesolve, onlyfiguredocumentmanagesolve)
- **pushrecommendcontinueadmit**: `VLMBaseModelAdapter + ModelSlimPipelineInterfaceV1` (`ModelInfoInterface` canselectbutSuggest)
- **mustmustImplementation** (5 count) : `handle_dataset`, `init_model`, `generate_model_visit`, `generate_model_forward`, `enable_kv_cache`
- **TemplatemiddleoftenseeassistaidMethod** (nonFrameworkstrongmake, butmultiplenumberModelrequires) : `generate_decoder_layer`, `_load_decoder_if_not_exist`, `_create_model_instance`
## specialspecialsituationsituation (requiressinglealonehandlemanage)
### LLM specialspecialsituationsituation
- **decoder Pathnotoneconsistent**: notonedefineis `model.layers`, alsocanabilityis `model.decoder.layers` orotherotherPath; mustmustmodify `_decoder_layer_prefix`.
- **layerstructurecreateparameternumberdifferencedifferent**: havesome block structurecreateadapternotconnectreceive `layer_idx`, needmodify `_load_decoder_if_not_exist` ofactualexampletransformmethods.
- **MoE packed Weights**: ifas 3D packed experts, needfirst unpack, againsubstituteexchangeaslinepropertylayerspecializedexpertmodelblock.
- **nonstandardstandardConfigurationcharacterparagraph**: ifno `num_hidden_layers` orcharacterparagraphnamenotsame, `init_model` needaccording toitemstandard config modifycompose.
### VLM specialspecialsituationsituation
- **Datamustmustfiguredocumentbecomepair**: `handle_dataset` requiressametimehave `text` and `image`, puredocumentthissamplethisnotAdaptationoughtTemplate.
- **looksense/documentthisPathdifferencedifferent**: Templatefakeset `model.visual` and `model.language_model.layers`, itemstandardModelcanabilitynotsame, needaccording toReal `modeling` modify.
- **mergematchlogiclogicnotcansetTemplate**: `generate_model_forward` middle image embeds noteinputRules (token id, bitplacecompilecode, mask) Modeldifferencedifferentlarge, mustmustpairalignofficialmethod forward.
- **text_config Structuredifferencedifferent**: ifnotkeepin `config.text_config`, needmodifyasModelactualactualdocumentthisConfigurationPathandSynchronizationlayernumbercharacterparagraph.
- **processor travelasdifferencedifferent**: notsameModel `AutoProcessor` ofoutputinputkeyand `apply_chat_template` returnreturncharacterparagraphnotsame, needaccording toRealreturnreturnvalueadjustadjust keys.
### InterfacefunctionabilityDescription (mustmustfallactualtoCode)
#### 1) `handle_dataset(dataset, device) -> List[Any]`
- **jobresponsibility**: handlereasonbegincalibratestandardsamplethisconvertbecomeModelcanstraightconnectdisappearcostofoutputinputcolumntable.
- **outputinput**: reasonbeginDataset (throughoftenisdocumentthis list) anditemstandardDevice.
- **Output**: `List[Any]`, eachcountunitelementcanstraightconnectUsed foronetimepreviousdirection (if `model(*data)` or `model(**data)`) .
- **ImplementationSuggest**: optimizefirstcomplexusebasetype tokenization abilityforce (if `_get_tokenized_data`) , keepcertifycharacterparagraphnameandModel forward parameternumberpairalign.
- **completedjudgedefine**: QuantizationProcessReadoughtcolumntableafter, noneedamountexternalDataconvertexchangeimmediatecanenterinput `generate_model_forward`.
#### 2) `init_model(device) -> nn.Module`
- **jobresponsibility**: InitializationandreturnreturncanparticipationQuantizationProcessofModelactualexample.
- **outputinput**: itemstandardDevice (NPU/CPU) .
- **Output**: `nn.Module` (`eval()` statusstate) .
- **ImplementationSuggest**: according toModelRealStructureLoad; largeModelcanadoptusedistributelayer/lazyLoad, confirmkeepaftercontinue visit/forward canvisitasktoitemstandardlayer.
- **completedjudgedefine**: returnreturnModelafter, `generate_model_visit` abilityiteratehistoryitemstandardQuantizationlayer, andpreviousdirectioncanExecute.
#### 3) `generate_model_visit(model) -> Generator[ProcessRequest, Any, None]`
- **jobresponsibility**: definemeaning“according towhatwhatsequenceorderiteratehistorywhichsomemodelblock”performIterativelayerhandlemanage.
- **outputinput**: InitializationafterofModel.
- **Output**: according tosequenceorder `yield ProcessRequest` (eachcount request Correspondingonecountwaithandlemanagemodelblock) .
- **ImplementationSuggest**: in order toReal decoder/block sequenceorderOutput, notjumplayer, notweightarrange; NamePathshouldcanonlyonedefinebitmodelblock.
- **completedjudgedefine**: produceoutputoflayerordercolumnand `generate_model_forward` oneoneCorresponding.
#### 4) `generate_model_forward(model, inputs) -> Generator[ProcessRequest, Any, None]`
- **jobresponsibility**: definemeaningand `visit` pairalignofdistributeparagraphpreviousdirection, Used forIterativelayercalibratestandard.
- **outputinput**: Model + singleitemcalibratestandardoutputinput.
- **Output**: according tosequenceorder `yield ProcessRequest` (PackagecontainoughtlayerExecuteallneedoutputinput) .
- **ImplementationSuggest**: layersequenceorder, distributeparagraphsideboundary, expandamounttransferdeliverPathand `generate_model_visit` strictformatoneconsistent.
- **completedjudgedefine**: sameonelayerin visit/forward ofsearchleadandlanguagemeaningcompleteallmatchmatch, notoutputappearerrorbit.
#### 5) `enable_kv_cache(model, need_kv_cache) -> None`
- **jobresponsibility**: statisticsoneControl KV Cache openrelated.
- **outputinput**: Modelactualexampleandarrangeyouopenrelated.
- **Output**: noreturnreturn (reasonregionmodifymodify) .
- **ImplementationSuggest**: optimizefirstcomplexusebasetype `_enable_kv_cache`; arrivefewconfirmkeepmaininterfereModel config middle `use_cache` bepositiveconfirmSet.
- **completedjudgedefine**: openrelatedafterModeltravelasandpreperiodoneconsistent, calibratestandardScenariosunderthroughoftencanrelatedclosein order todowngradelowMemoryoccupyuse.
## relatedkeyImplementationreasonrule
### 1) `generate_model_visit` and `generate_model_forward` mustmuststrictformatoneconsistent
- iteratehistorylayersetmatchoneconsistent
- sequenceorderoneconsistent
- distributelayeroutputinputOutputtransferdeliveroneconsistent
thisismostcontenteasyoutputerror, alsomostshadowloudQuantizationpositiveconfirmpropertyofpartdistribute.
### 2) notneedrelyModelnameguessStructure
mustmustin order toReal `modeling` Codeasstandard, ConfirmlayerPath, commandnameand forward travelasafteragaincomposeAdaptationadapter.
### 3) VLM onlywalk“looksenseadjustbody + documentthisIterativelayer”
- optimizefirstcomplexuse VLM basetype
- visit/forward middlelooksensemodelblockanddocumentthislayersequenceorderkeepmaintainoneconsistent
- figuredocumentmergematchlogiclogicneedpairalignitemstandardModelofficialmethod forward
### 4) MoE mergematchStructureoptimizefirstaccording to“unpack afterpurelinepropertylayer”Adaptation
verymultiplenewModelof MoE Usagemergematch/printPackageWeights (oftenseeas 3D expandamount) , andQuantizationandIterativelayerhandlemanagethroughoftenupdatesuitablematch `nn.Linear` shapeformulaofspecializedexpertImplementation.
ImplementationRequirements:
- firstjudgejudgereasonbeginImplementationiswhetheras 3D packed experts (notneedfakesetallhave MoE allonesample)
- ifis packed Structure, notonlyneedinLoadtime unpack, alsoneedImplementationCorrespondingof MoE splitdistribute module
- unpack afterspecializedexpertshouldfalltopurelinepropertylayer (`gate_proj` / `up_proj` / `down_proj`) , avoidavoidaftercontinueProcessstraightconnectaccordingdepend 3D Weights
- pushrecommendStructure: `moe_utils.py` Providessplitdistributeafterof MoE module, `model_adapter.py` negativeresponsibilityWeights unpack andmodelblocksubstituteexchange
canReference `qwen3_5` ofImplementationthinkpath (`moe_utils.py`, `modeling_qwen3_5_mtp.py`) : firstrecognizecategory packed Weights, againsplitdistributeasIterative expert linepropertylayer.
ExamplesCodepleaseReference:
- `references/moe_unpacked_module_example.py`
- `references/moe_unpacked_adapter_example.py`
references/interface_checklist.md
# mustneedInterfaceCheckChecklist
inrunVerificationprevious, Confirmin order tounderMethodalreadyImplementationandCorrect Integration:
- [ ] `handle_dataset`
- [ ] `init_model`
- [ ] `generate_model_visit`
- [ ] `generate_model_forward`
- [ ] `enable_kv_cache`
## pairalignCheck
- [ ] `generate_model_visit` and `generate_model_forward` iteratehistoryoflayeroneconsistent
- [ ] iteratehistorysequenceorderoneconsistent
- [ ] layerbetweenoutputinputOutputtransferdeliveroneconsistent
## RegistrationCheck
- [ ] `config/config.ini` of `[ModelAdapter]` underalreadyConfigurationModelcategoryname
- [ ] `config/config.ini` of `[ModelAdapterEntryPoints]` underalreadyConfigurationEntry
- [ ] CodemodifymodifyafteralreadyweightnewInstallationPackage
references/interface_reference.md
# ModelAdaptationBase InterfaceReference
thisDocumentsKeep OnlyModelAdaptationDevelopmentRequiredBase Interface.
Does Not Include SmoothQuant, QuaRot, FA3, FlatQuant etchighlevelcomputemethodInterface.
## 1) IModel (BaseModelattributeproperty)
**bitplace**: `msmodelslim/model/interface.py`
allhaveAdaptationadapterofBaseattributepropertyInterface:
```python
class IModel:
@property
def model_type(self) -> str
@property
def model_path(self) -> Path
@property
def trust_remote_code(self) -> bool
```
ImplementationRequirements:
- `model_type`: returnreturnModeltypetypestandardrecognize.
- `model_path`: returnreturnModelDirectoryPath.
- `trust_remote_code`: returnreturniswhetherallowallowfarprocessCode.
## 2) ModelSlimPipelineInterfaceV1 (mustneed)
**bitplace**: `msmodelslim/core/runner/pipeline_interface.py`
BaseQuantizationAdaptationmustmustImplementationofCoreInterface:
```python
class PipelineInterface(IModel):
@abstractmethod
def handle_dataset(self, dataset: Any, device: DeviceType = DeviceType.NPU) -> List[Any]:
...
@abstractmethod
def init_model(self, device: DeviceType = DeviceType.NPU) -> nn.Module:
...
@abstractmethod
def generate_model_visit(self, model: nn.Module) -> Generator[ProcessRequest, Any, None]:
...
@abstractmethod
def generate_model_forward(self, model: nn.Module, inputs: Any) -> Generator[ProcessRequest, Any, None]:
...
@abstractmethod
def enable_kv_cache(self, model: nn.Module, need_kv_cache: bool) -> None:
...
```
Implementationweightpoint:
- `generate_model_visit` and `generate_model_forward` oflayersequenceordermustmuststrictformatoneconsistent.
- `handle_dataset` OutputmustmustcanstraightconnectUsed forpreviousdirection.
- `init_model` returnreturncanExecutepreviousdirectionandcanbeIterativelayervisitaskofModel.
## 3) ModelInfoInterface (pushrecommend)
**bitplace**: `msmodelslim/app/naive_quantization/model_info_interface.py`
(partdistributeScenariosalsoin `msmodelslim/app/auto_tuning/model_info_interface.py` Usage)
Used forProvidesModelBaseinformationinformation:
```python
def get_model_pedigree(self) -> str
def get_model_type(self) -> str
```
Description:
- oughtInterfacethroughoftenand `TransformersModel + ModelSlimPipelineInterfaceV1` groupmatchUsage.
- ifyouofAdaptationProcessorguideoutputProcessaccordingdependModelexpertfamilyinformationinformation, SuggestImplementation.
## pushrecommendcontinueadmitgroupmatch
BaseModelAdaptation (LLM/VLM documentthismaininterfere) Suggest:
```python
class MyModelAdapter(TransformersModel,
ModelInfoInterface,
ModelSlimPipelineInterfaceV1):
pass
```
ifwhenpreviousScenariosnotrequiresModelinformationinformationabilityforce, cansavestrategy `ModelInfoInterface`, but `ModelSlimPipelineInterfaceV1` notcansavestrategy.
references/llm/fallback_config.yaml
apiversion: modelslim_v1
spec:
process:
- type: "linear_quant"
qconfig:
act:
scope: "per_token"
dtype: "int8"
symmetric: True
method: "minmax"
weight:
scope: "per_channel"
dtype: "int8"
symmetric: True
method: "minmax"
include: ["*"]
exclude: ["*"] # Full Fallback，Not ActualQuantizationAny Layer
save:
- type: "ascendv1_saver"
part_file_size: 4
references/llm/w8a8_dynamic_full_model.yaml
apiversion: modelslim_v1
spec:
process:
- type: "linear_quant"
qconfig:
act:
scope: "per_token" # dynamic
dtype: "int8"
symmetric: True
method: "minmax"
weight:
scope: "per_channel"
dtype: "int8"
symmetric: True
method: "minmax"
include: ["*"] # allModelQuantization
exclude: [] # notFallback
save:
- type: "ascendv1_saver"
part_file_size: 4
references/llm/w8a8_static_full_model.yaml
apiversion: modelslim_v1
spec:
process:
- type: "linear_quant"
qconfig:
act:
scope: "per_tensor" # static
dtype: "int8"
symmetric: False
method: "minmax"
weight:
scope: "per_channel"
dtype: "int8"
symmetric: True
method: "minmax"
include: ["*"] # allModelQuantization
exclude: [] # notFallback
save:
- type: "ascendv1_saver"
part_file_size: 4
references/model_analysis.md
# ModelStructureAnalysisGuide
## 1. ConfirmModelStructure Source
- Read `config.json` of `model_type`, `architectures`, `auto_map`
- Determine Structure SourceModelRepository `modeling_*.py` alsois transformers officialmethodImplementation
## 2. definebitandreviewreadModelImplementation (mustmust)
- **CustomImplementation**: ifresult `auto_map` fingerdirectionCustomImplementation (if `modeling_xxx.XXXForCausalLM`) , optimizefirstreviewreadModelDirectorymiddleof `modeling_*.py`
- **officialmethodImplementation**: ifresultUsage transformers officialmethodImplementation, throughoftenin:
- sourcecodePath: `transformers/src/transformers/models/<model_type>/modeling_<model_type>.py`
- guideinputPath: `transformers.models.<model_type>.modeling_<model_type>`
- **weightpointreviewread**:
- DecoderLayer definemeaning
- attention/MLP commandname
- `forward` inputparticipationreturnreturnvalue
- **MoE ModelamountexternalCheck**:
- `experts` Weightsiswhetheras 3D packed Structure (if `experts.gate_up_proj` / `experts.down_proj`)
references/moe_unpacked_adapter_example.py
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
"""
Adapter-side unpack example for MoE fused weights.
Goal:
- Detect whether experts are packed 3D tensors.
- Unpack packed weights into per-expert Linear parameter keys.
- Replace original MoE block with unpacked Linear-expert module.
"""
from typing import Dict
import torch
from .moe_unpacked_module_example import SparseMoeBlockWithLinearExperts
def is_packed_moe_tensor(key: str, tensor: torch.Tensor) -> bool:
"""Heuristic: packed experts are usually 3D for gate_up/down projections."""
if not isinstance(tensor, torch.Tensor):
return False
if tensor.dim() != 3:
return False
return key.endswith("experts.gate_up_proj") or key.endswith("experts.down_proj")
def unpack_packed_moe_weights(state_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
"""
Unpack common packed MoE keys into Linear-expert keys.
Expected packed patterns:
- *.experts.gate_up_proj: [num_experts, 2*intermediate, hidden]
- *.experts.down_proj: [num_experts, hidden, intermediate]
"""
unpacked = dict(state_dict)
for key, tensor in state_dict.items():
if not is_packed_moe_tensor(key, tensor):
continue
if key.endswith("experts.gate_up_proj"):
prefix = key.replace("experts.gate_up_proj", "experts.")
num_experts = tensor.shape[0]
for i in range(num_experts):
gate_w, up_w = tensor[i].chunk(2, dim=0)
unpacked[f"{prefix}{i}.gate_proj.weight"] = gate_w
unpacked[f"{prefix}{i}.up_proj.weight"] = up_w
elif key.endswith("experts.down_proj"):
prefix = key.replace("experts.down_proj", "experts.")
num_experts = tensor.shape[0]
for i in range(num_experts):
unpacked[f"{prefix}{i}.down_proj.weight"] = tensor[i]
return unpacked
def replace_moe_module_if_needed(layer, cfg, act_fn):
"""
Replace fused MoE module with unpacked Linear-expert module.
Call this during layer construction/loading in model_adapter.py.
"""
if not hasattr(layer, "mlp") or not hasattr(layer.mlp, "experts"):
return layer
# Example: original layer.mlp uses packed expert layout.
unpacked_moe = SparseMoeBlockWithLinearExperts(
hidden_size=cfg.hidden_size,
intermediate_size=cfg.moe_intermediate_size,
num_experts=cfg.num_experts,
top_k=cfg.num_experts_per_tok,
act_fn=act_fn,
)
layer.mlp = unpacked_moe
return layer
def load_layer_with_unpack_example(layer, layer_state_dict: Dict[str, torch.Tensor], strict: bool = False):
"""
Example load sequence inside adapter:
1) unpack packed MoE tensors
2) load state dict into replaced module structure
"""
unpacked_state_dict = unpack_packed_moe_weights(layer_state_dict)
missing, unexpected = layer.load_state_dict(unpacked_state_dict, strict=strict)
return {
"missing_keys": list(missing),
"unexpected_keys": list(unexpected),
"loaded_keys": len(unpacked_state_dict),
}
references/moe_unpacked_module_example.py
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
"""
MoE unpacked module example.
Purpose:
- Provide a pure-nn.Linear MoE implementation for models whose original experts
are fused/packed in 3D tensors.
- Keep routing logic and forward behavior consistent with the source model.
Note:
- This is an adaptation example, not a drop-in module for every model.
- You must align tensor layouts and routing semantics with the original modeling file.
"""
from typing import Callable
import torch
import torch.nn as nn
import torch.nn.functional as F
class MoeExpertMLP(nn.Module):
"""Single expert with pure Linear layers."""
def __init__(self, hidden_size: int, intermediate_size: int, act_fn: Callable[[torch.Tensor], torch.Tensor]):
super().__init__()
self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=False)
self.act_fn = act_fn
def forward(self, x: torch.Tensor) -> torch.Tensor:
gate = self.act_fn(self.gate_proj(x))
up = self.up_proj(x)
return self.down_proj(gate * up)
class TopKRouter(nn.Module):
"""Generic top-k router example."""
def __init__(self, hidden_size: int, num_experts: int, top_k: int):
super().__init__()
self.top_k = top_k
self.weight = nn.Parameter(torch.empty((num_experts, hidden_size)))
def forward(self, hidden_states: torch.Tensor):
logits = F.linear(hidden_states, self.weight)
probs = torch.softmax(logits, dim=-1, dtype=torch.float)
topk_prob, topk_idx = torch.topk(probs, self.top_k, dim=-1)
topk_prob = topk_prob / topk_prob.sum(dim=-1, keepdim=True)
return logits, topk_prob.to(logits.dtype), topk_idx
class SparseMoeBlockWithLinearExperts(nn.Module):
"""
MoE block where each expert is explicit Linear layers.
This is the target structure after unpack.
"""
def __init__(
self,
hidden_size: int,
intermediate_size: int,
num_experts: int,
top_k: int,
act_fn: Callable[[torch.Tensor], torch.Tensor],
):
super().__init__()
self.num_experts = num_experts
self.router = TopKRouter(hidden_size=hidden_size, num_experts=num_experts, top_k=top_k)
self.experts = nn.ModuleList(
[MoeExpertMLP(hidden_size, intermediate_size, act_fn) for _ in range(num_experts)]
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
batch_size, seq_len, hidden_size = hidden_states.shape
flat = hidden_states.reshape(-1, hidden_size)
_, routing_weights, selected_experts = self.router(flat)
out = torch.zeros_like(flat)
expert_mask = F.one_hot(selected_experts, num_classes=self.num_experts).permute(2, 1, 0)
active_experts = torch.nonzero(expert_mask.sum(dim=(-1, -2)) > 0).flatten()
for expert_idx in active_experts.tolist():
top_k_pos, token_idx = torch.where(expert_mask[expert_idx])
expert_out = self.experts[expert_idx](flat[token_idx])
expert_out = expert_out * routing_weights[token_idx, top_k_pos, None]
out.index_add_(0, token_idx, expert_out.to(out.dtype))
return out.reshape(batch_size, seq_len, hidden_size)
references/registration_guide.md
# AdaptationAdapter RegistrationGuide
in `config/config.ini` middleRegistrationModelandEntry.
## Examples
```ini
[ModelAdapter]
my_model = MyModel-7B, MyModel-13B
[ModelAdapterEntryPoints]
my_model = msmodelslim.model.my_model.model_adapter:MyModelAdapter
```
Registrationcompletedafter, Must Execute `bash install.sh` InstallationUpdate.
references/troubleshooting.md
# Troubleshooting
## 1. Model Loading Issues
### Issue: transformers version incompatibility
**Symptom:** `ImportError: cannot import name 'xxx' from 'transformers'`
**Root Cause:** Model requires newer transformers version.
**Solution:**
```bash
pip install --upgrade transformers
# Or specific version
pip install transformers>=4.40.0
```
### Issue: trust_remote_code not set
**Symptom:** `OSError: xxx requires trust_remote_code=True`
**Solution:**
```python
from transformers import AutoModel, AutoTokenizer
model = AutoModel.from_pretrained(path, trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True)
```
## 2. Adapter Creation Issues
### Issue: Missing required interfaces
**Symptom:** `NotImplementedError: Interface xxx not implemented`
**Solution:** Ensure all 5 required interfaces are implemented:
- handle_dataset
- init_model
- generate_model_visit
- generate_model_forward
- enable_kv_cache
### Issue: visit and forward mismatch
**Symptom:** Quantization fails with layer mismatch
**Solution:** Verify generate_model_visit and generate_model_forward process layers in the same order and access the same modules.
## 3. MoE Weight Issues
### Issue: Packed expert weights cause quantization failure
**Symptom:** Shape mismatch during quantization
**Root Cause:** MoE experts packed as 3D tensors [num_experts, hidden, intermediate]
**Solution:** Unpack before quantization:
```python
# For weights with shape [num_experts, hidden, intermediate*3]
gate = weights[0::3] # [num_experts, hidden, intermediate]
up = weights[1::3] # [num_experts, hidden, intermediate]
down = weights[2::3] # [num_experts, hidden, intermediate]
```
### Issue: Incorrect unpack dimension
**Symptom:** Weight shapes don't match after unpack
**Solution:** Check the actual weight dimension ordering. Some models use [hidden, intermediate, num_experts].
## 4. Registration Issues
### Issue: Model not found after install.sh
**Symptom:** `KeyError: model_type not found in config.ini`
**Solution:**
```bash
# Verify config.ini entries
grep -A2 "\[ModelAdapter\]" config/config.ini
grep -A2 "\[ModelAdapterEntryPoints\]" config/config.ini
# Re-run installation
bash install.sh
# Verify Python can import
python3 -c "from adapters.<model_name> import model_adapter"
```
### Issue: Module import error
**Symptom:** `ModuleNotFoundError: No module named 'adapters'`
**Solution:**
```bash
# Check __init__.py exists
ls -la adapters/__init__.py
# Re-install package
pip install -e .
```
## 5. Quantization Issues
### Issue: Step 3 weight verification failed
**Symptom:** `AssertionError: Max tolerance exceeded`
**Possible Causes:**
1. Quantization changed weight values unexpectedly
2. Different tensor ordering between fallback and original
**Solution:**
```bash
# Check actual tolerance
python3 scripts/step3_verify_weights.py --model-path ./test_model --ref-path ./models/<model_name> --verbose
# If using MoE, verify unpack logic
# Increase tolerance if within acceptable range (e.g., 1e-3)
```
### Issue: Step 4 quant description validation failed
**Symptom:** Layer names don't match rules
**Solution:**
```bash
# Check which layers failed
python3 scripts/step4_verify_quant_description.py --model-path ./test_model --rules-path ./rules.json --verbose
# Update rules.json to match actual layer names
```
## 6. Performance Issues
### Issue: Model loading too slow
**Solution:**
- Use modelscope download for non-weight files only
- Enable layer-by-layer loading for large models
- Use trust_remote_code=False if not needed
### Issue: Quantization out of memory
**Solution:**
- Reduce batch size in calibration data
- Use dynamic quantization instead of static
- Process in smaller chunks
## Quick Diagnostic Commands
```bash
# Check Python environment
python3 --version && pip list | grep -E "transformers|torch|msmodelslim"
# Verify model structure
python3 -c "import json; c=json.load(open('models/<model>/config.json')); print(c.get('model_type'))"
# Test adapter import
python3 -c "from adapters.<model> import model_adapter; print('Import OK')"
# Run verification with verbose
python3 scripts/step1_generate_test_model.py --model-path ./models/<model> --output ./test --verbose
```
references/verification_guide.md
# AdaptationadapterVerificationGuide
## CoreVerificationProcess (mustmust)
mustmustExecute in Orderin order tounderfourstepVerification:
1. **GenerateTestingModel** (Step 1)
- VerificationModelLoadandBasicConfiguration
- Generate Random WeightsofsmalltypeModelUsed forrapidspeedTesting
2. **allFallbackQuantization** (Step 2)
- VerificationQuantizationProcessiswhetherabilityrunthrough (notinvolveandtoolbodyprecisiondegree, onlyrunthroughProcess)
- Check `model_adapter` Registrationiswhetherlivevalid
3. **allFallbackModeloneconsistentpropertyandcanLoad/keepkeepVerification** (Step 3)
- based on Step2 GenerateofallFallbackModel, Verificationotherand Step1 floatpointModelWeightsstrictformatoneconsistent (key, shapestatus, numbervalue)
- VerificationoughtModelproduceobjecttoolpreparecompleteadjustLoad/keepkeepabilityforce (canbeaftercontinueProcessReadandcontinuecontinuehandlemanage)
4. **actualactualQuantizationProcessVerification** (Step 4)
- Runactualactual W8A8 quietstate/movestateQuantizationProcess (nonFallbackProcess) andproduceoutputQuantizationResult
- VerificationQuantizationDescriptionfilewhether matchespreperiodRules, ChecklinepropertylayerQuantizationstandardsigniswhetherpositiveconfirm
## Verificationcommandcommand
```bash
# 1) GenerateTestingModel
python scripts/step1_generate_test_model.py \
--model-path /path/to/your/model \
--output-path /tmp/test_model
# 2) allFallbackQuantization
python scripts/step2_run_quantization.py \
--model-path /tmp/test_model \
--output-path /tmp/quantized_model \
--model-type YourModelType \
--model-family llm
# multiplemodelstateModelpleaseUsage:
# --model-family vlm
# 3) allFallbackModeloneconsistentpropertyVerification (andfloatpointWeightsstrictformatpairalign)
python scripts/step3_verify_weights.py \
--original-path /tmp/test_model \
--quantized-path /tmp/quantized_model \
--tolerance 1e-5
```
### Step 4: allModelQuantizationCheck
ExecuteallModel W8A8 quietstateQuantizationandCheckDescriptionfile:
```bash
# ExecuteQuantization
msmodelslim quant \
--model_type <your_model_type> \
--model_path /tmp/test_model \
--save_path /tmp/quantized_w8a8_static \
--device cpu \
--config_path references/llm/w8a8_static_full_model.yaml \
--trust_remote_code True
# VerificationDescriptionfile
python scripts/step4_verify_quant_description.py \
--desc-path /tmp/quantized_w8a8_static \
--rules-path /path/to/your_verify_rules_static.json
```
ExecuteallModel W8A8 movestateQuantizationandCheckDescriptionfile:
```bash
# ExecuteQuantization
msmodelslim quant \
--model_type <your_model_type> \
--model_path /tmp/test_model \
--save_path /tmp/quantized_w8a8_dynamic \
--device cpu \
--config_path references/llm/w8a8_dynamic_full_model.yaml \
--trust_remote_code True
# VerificationDescriptionfile
python scripts/step4_verify_quant_description.py \
--desc-path /tmp/quantized_w8a8_dynamic \
--rules-path /path/to/your_verify_rules_dynamic.json
```
multiplemodelstateModel (VLM) SuggestUsagein order tounderConfigurationTemplate (containcalibratestandardDatacharacterparagraph) :
```bash
references/vlm/w8a8_static_full_model.yaml
references/vlm/w8a8_dynamic_full_model.yaml
```
Description: notagaininsideplace `verify_rules_w8a8_static.json` / `verify_rules_w8a8_dynamic.json`, please agent according toitemstandardModellayernameselftravelGenerateRules Fileandtransferinput `--rules-path`.
## throughexceedstandardstandard
- **CoreVerification**: Step 1/2/3/4 averagebecomefunctionExecutenoreporterror.
- **Step 3 throughexceedPrerequisites**: allFallbackModelandfloatpointModelWeightsCheck PASS, andQuantizationproduceobjectcanbeaftercontinueProcesspositiveoftenLoad/Usage.
- **Step 4 throughexceedPrerequisites**: actualactualQuantizationProcessExecutebecomefunction, DescriptionfileRulescalibrateverifythroughexceed.
## rapidspeedarrangeerror / lossfailuredistributeflow
- **Step 1 lossfailure**:
- ModelLoadlossfailure: Check `transformers` Versionor `trust_remote_code` Set
- typetypenotSupport: Check `model_type` iswhetherinSupportcolumntablemiddle
- **Step 2 lossfailure**:
- findnottoAdaptationadapter: Check `config.ini` Registrationiswhetherpositiveconfirm, iswhetherExecutecompleted `install.sh`
- QuantizationEntryreporterror: Check `handle_dataset` Datahandlemanageiswhetherpositiveconfirm
- **Step 3 lossfailure (allFallbackModelandfloatpointnotoneconsistent/notcancompleteadjustLoad)**:
- CheckQuantizationbefore and afterWeightskeyname, shapestatusandreflectshootrelatedsystem (shouldoneoneCorresponding)
- Checknumbervaluedifferencedifferentiswhetherexceedoutputthresholdvalue (silentrecognize `tolerance=1e-5`)
- CheckQuantizationDirectoryinsideWeightsandmustneedConfigurationfileiswhethercompleteadjust, confirmkeepcanbeaftercontinueProcessRead
- **MoE Model**: ifUsage packed Weights, Check `packed -> unpacked` splitdistributelogiclogiciswhetherpositiveconfirm (dimensiondegree, convertplace)
- **Step 4 lossfailure (actualactualQuantizationProcessorDescriptionfiledifferentoften)**:
- CheckactualactualQuantizationConfigurationiswhetherpositiveconfirm (W8A8 quietstate/movestate, calibratestandardparameternumberetc)
- CheckiswhethererrorusecompletedFallbackConfiguration
- CheckVerificationRules JSON middleofrelatedkeycharacteriswhethercovercovercompletedModelactualactuallayername
references/verification-method.md
# Verification Methods
## Prerequisite Verification
### 1. Verify Python Environment
```bash
python3 --version # Python >= 3.8
pip list | grep transformers # transformers installed
pip list | grep msmodelslim # msmodelslim installed
```
### 2. Verify Model Files
```bash
# Check model directory structure
ls -la models/<model_name>/
# Expected: config.json, modeling_*.py, configuration_*.py
# Verify config.json exists
cat models/<model_name>/config.json | grep model_type
```
### 3. Verify Adapter Registration
```bash
# Check config.ini entries
grep -A5 "\[ModelAdapter\]" config/config.ini
grep -A5 "\[ModelAdapterEntryPoints\]" config/config.ini
# Test import
python3 -c "from adapters.<model_name> import model_adapter; print('OK')"
```
## Functional Verification
### 1. Model Analysis Verification
```bash
# Read config.json
cat models/<model_name>/config.json
# Check model_type
python3 -c "import json; c=json.load(open('models/<model_name>/config.json')); print(c.get('model_type'))"
# Check architectures
python3 -c "import json; c=json.load(open('models/<model_name>/config.json')); print(c.get('architectures'))"
```
### 2. Adapter Creation Verification
```bash
# Step 1: Generate test model
python3 scripts/step1_generate_test_model.py --model-path ./models/<model_name> --output ./test_model
# Step 2: Run quantization (fallback)
python3 scripts/step2_run_quantization.py --model-path ./test_model --config ./references/llm/fallback_config.yaml
# Step 3: Verify weights
python3 scripts/step3_verify_weights.py --model-path ./test_model --ref-path ./models/<model_name>
# Step 4: Verify quant description
python3 scripts/step4_verify_quant_description.py --model-path ./test_model --rules-path ./rules.json
```
### 3. Interface Implementation Verification
```python
# Test all required interfaces
from adapters.<model_name>.model_adapter import <ModelName>Adapter
adapter = <ModelName>Adapter()
# Test handle_dataset
data = adapter.handle_dataset(["test input"], "cuda:0")
assert data is not None, "handle_dataset failed"
# Test init_model
import json
config = json.load(open("models/<model_name>/config.json"))
model = adapter.init_model(config, "cuda:0")
assert model is not None, "init_model failed"
# Test generate_model_visit
visit = adapter.generate_model_visit()
assert len(visit) > 0, "generate_model_visit failed"
# Test generate_model_forward
forward = adapter.generate_model_forward()
assert forward is not None, "generate_model_forward failed"
# Test enable_kv_cache
adapter.enable_kv_cache(model)
print("All interfaces verified successfully")
```
## End-to-End Verification Script
```bash
#!/bin/bash
set -e
MODEL_NAME="Qwen3-14B"
MODEL_PATH="./models/${MODEL_NAME}"
echo "=== 1. Verify Prerequisites ==="
python3 --version
pip list | grep -E "transformers|msmodelslim"
echo "=== 2. Verify Model Files ==="
ls -la ${MODEL_PATH}/config.json
echo "=== 3. Run Step 1: Generate Test Model ==="
python3 scripts/step1_generate_test_model.py --model-path ${MODEL_PATH} --output ./test_model
echo "=== 4. Run Step 2: Full Fallback Quantization ==="
python3 scripts/step2_run_quantization.py --model-path ./test_model --config ./references/llm/fallback_config.yaml
echo "=== 5. Run Step 3: Weight Verification ==="
python3 scripts/step3_verify_weights.py --model-path ./test_model --ref-path ${MODEL_PATH}
echo "=== 6. Run Step 4: Quant Description ==="
python3 scripts/step4_verify_quant_description.py --model-path ./test_model --rules-path ./rules.json
echo "=== All verifications passed ==="
```
## Verification Checklist
| Check | Expected Result |
|-------|-----------------|
| Python version | >= 3.8 |
| transformers installed | Import successful |
| msmodelslim installed | Import successful |
| config.json exists | File readable |
| model_type identified | Valid type string |
| Step 1 success | Test model generated |
| Step 2 success | Quantization completed |
| Step 3 tolerance | <= 1e-5 |
| Step 4 success | Description validated |
| All interfaces | Import and call successful |
references/vlm/fallback_config.yaml
apiversion: multimodal_vlm_modelslim_v1
spec:
process:
- type: "linear_quant"
qconfig:
act:
scope: "per_token"
dtype: "int8"
symmetric: True
method: "minmax"
weight:
scope: "per_channel"
dtype: "int8"
symmetric: True
method: "minmax"
include: ["*"]
exclude: ["*"] # Full Fallback，Not ActualQuantizationAny Layer
save:
- type: "ascendv1_saver"
part_file_size: 4
dataset: "calibImages"
default_text: "Describe this image in detail."
references/vlm/w8a8_dynamic_full_model.yaml
apiversion: multimodal_vlm_modelslim_v1
spec:
process:
- type: "linear_quant"
qconfig:
act:
scope: "per_token" # dynamic
dtype: "int8"
symmetric: True
method: "minmax"
weight:
scope: "per_channel"
dtype: "int8"
symmetric: True
method: "minmax"
include: ["*"] # allModelQuantization
exclude: [] # notFallback
save:
- type: "ascendv1_saver"
part_file_size: 4
# multiplemodelstatecalibratestandardData (needaccording toactualactualDatasetNameadjustadjust)
dataset: "calibImages"
default_text: "Describe this image in detail."
references/vlm/w8a8_static_full_model.yaml
apiversion: multimodal_vlm_modelslim_v1
spec:
process:
- type: "linear_quant"
qconfig:
act:
scope: "per_tensor" # static
dtype: "int8"
symmetric: False
method: "minmax"
weight:
scope: "per_channel"
dtype: "int8"
symmetric: True
method: "minmax"
include: ["*"] # allModelQuantization
exclude: [] # notFallback
save:
- type: "ascendv1_saver"
part_file_size: 4
# multiplemodelstatecalibratestandardData (needaccording toactualactualDatasetNameadjustadjust)
dataset: "calibImages"
default_text: "Describe this image in detail."
scripts/step1_generate_test_model.py
#!/usr/bin/env python3
"""Steps1: Generate Random WeightsTestingModel (Slim Version) . """
import argparse
import json
import os
import shutil
import sys
import torch
from transformers import AutoConfig
import transformers
def _read_json(path):
with open(path, "r", encoding="utf-8") as f:
return json.load(f)
def _write_json(path, data):
with open(path, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
def _copy_non_weight_files(src_dir, dst_dir):
os.makedirs(dst_dir, exist_ok=True)
for name in os.listdir(src_dir):
src = os.path.join(src_dir, name)
dst = os.path.join(dst_dir, name)
if os.path.isdir(src):
continue
if name.endswith(".safetensors"):
continue
if name.endswith(".index.json"):
continue
shutil.copy2(src, dst)
def _shrink_config(cfg, num_layers):
out = dict(cfg)
if "text_config" in out:
text_cfg = dict(out["text_config"])
text_cfg["num_hidden_layers"] = num_layers
if isinstance(text_cfg.get("layer_types"), list):
text_cfg["layer_types"] = text_cfg["layer_types"][:num_layers]
out["text_config"] = text_cfg
else:
out["num_hidden_layers"] = num_layers
if isinstance(out.get("layer_types"), list):
out["layer_types"] = out["layer_types"][:num_layers]
return out
def _build_random_model_from_config(config):
candidate_auto_model_names = [
"AutoModelForCausalLM",
"AutoModelForImageTextToText",
"AutoModel",
]
errors = []
for cls_name in candidate_auto_model_names:
auto_cls = getattr(transformers, cls_name, None)
if auto_cls is None:
continue
try:
model = auto_cls.from_config(
config, trust_remote_code=True, torch_dtype=torch.float32
)
return model, cls_name
except Exception as e: # pragma: no cover - best-effort fallback chain
errors.append(f"{cls_name}: {repr(e)}")
raise RuntimeError(
"Cannot build model fromConfigurationBuildModel. Attempted: "
+ ", ".join(candidate_auto_model_names)
+ "\nError details:\n"
+ "\n".join(errors)
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--model-path", required=True)
parser.add_argument("--output-path", required=True)
parser.add_argument("--num-layers", type=int, default=2)
parser.add_argument("--device", default="cpu")
args = parser.parse_args()
src_cfg = os.path.join(args.model_path, "config.json")
if not os.path.exists(src_cfg):
print(f"[ERROR] MissingConfigurationfile: {src_cfg}")
return 1
_copy_non_weight_files(args.model_path, args.output_path)
cfg = _read_json(src_cfg)
_write_json(
os.path.join(args.output_path, "config.json"),
_shrink_config(cfg, args.num_layers),
)
config = AutoConfig.from_pretrained(args.output_path, trust_remote_code=True)
model, used_cls_name = _build_random_model_from_config(config)
print(f"[INFO] UsageModeltype: {used_cls_name}")
model = model.to(args.device)
model.train(False) # Set inference mode (disable dropout/batchnorm)
model.save_pretrained(args.output_path)
stale_index = os.path.join(args.output_path, "model.safetensors.index.json")
if os.path.exists(stale_index) and os.path.exists(
os.path.join(args.output_path, "model.safetensors")
):
os.remove(stale_index)
print(f"[OK] step1completed: {args.output_path}")
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/step2_run_quantization.py
#!/usr/bin/env python3
"""Steps2: Execute Full FallbackQuantization (Slim Version, Support LLM/VLM) . """
import argparse
import os
import subprocess
import sys
LLM_FALLBACK_YAML = """apiversion: modelslim_v1
spec:
process:
- type: "linear_quant"
qconfig:
act:
scope: "per_token"
dtype: "int8"
symmetric: True
method: "minmax"
weight:
scope: "per_channel"
dtype: "int8"
symmetric: True
method: "minmax"
include: ["*"]
exclude: ["*"]
save:
- type: "ascendv1_saver"
part_file_size: 4
"""
VLM_FALLBACK_YAML = """apiversion: multimodal_vlm_modelslim_v1
spec:
process:
- type: "linear_quant"
qconfig:
act:
scope: "per_token"
dtype: "int8"
symmetric: True
method: "minmax"
weight:
scope: "per_channel"
dtype: "int8"
symmetric: True
method: "minmax"
include: ["*"]
exclude: ["*"]
save:
- type: "ascendv1_saver"
part_file_size: 4
dataset: "calibImages"
default_text: "Describe this image in detail."
"""
def _write_fallback_yaml(path, model_family: str):
os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
content = VLM_FALLBACK_YAML if model_family == "vlm" else LLM_FALLBACK_YAML
with open(path, "w", encoding="utf-8") as f:
f.write(content)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--model-path", required=True)
parser.add_argument("--output-path", required=True)
parser.add_argument("--model-type", required=True)
parser.add_argument("--device", default="cpu")
parser.add_argument("--config-path", default="")
parser.add_argument("--model-family", choices=["llm", "vlm"], default="llm")
args = parser.parse_args()
config_path = args.config_path or os.path.join(args.output_path, "fallback_config.yaml")
if not os.path.exists(config_path):
_write_fallback_yaml(config_path, args.model_family)
os.makedirs(args.output_path, exist_ok=True)
cmd = [
sys.executable,
"-m",
"msmodelslim",
"quant",
"--model_path",
args.model_path,
"--save_path",
args.output_path,
"--device",
args.device,
"--model_type",
args.model_type,
"--config_path",
config_path,
"--trust_remote_code",
"True",
]
rc = subprocess.run(cmd, check=False).returncode
if rc != 0:
print("[ERROR] step2lossfailure")
return rc
print(f"[OK] step2completed: {args.output_path}")
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/step3_verify_weights.py
#!/usr/bin/env python3
"""Steps3: VerificationWeight Consistency (Slim Version) . """
import argparse
import glob
import os
import sys
import torch
def _load_weights(model_path):
try:
from safetensors.torch import load_file
files = sorted(glob.glob(os.path.join(model_path, "*.safetensors")))
if files:
merged = {}
for file in files:
merged.update(load_file(file))
return merged
except Exception:
pass
pt_path = os.path.join(model_path, "pytorch_model.bin")
if os.path.exists(pt_path):
return torch.load(pt_path, map_location="cpu")
return {}
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--original-path", required=True)
parser.add_argument("--quantized-path", required=True)
parser.add_argument("--tolerance", type=float, default=1e-5)
args = parser.parse_args()
left = _load_weights(args.original_path)
right = _load_weights(args.quantized_path)
if not left or not right:
print("[ERROR] step3lossfailure: WeightsLoadlossfailure")
return 1
left_keys = set(left.keys())
right_keys = set(right.keys())
if left_keys != right_keys:
print("[ERROR] step3lossfailure: Weightskeynotoneconsistent")
print(f"[INFO] onlyleftsidenumberamount: {len(left_keys - right_keys)}")
print(f"[INFO] onlyrightsidenumberamount: {len(right_keys - left_keys)}")
return 1
max_diff = 0.0
for key in sorted(left_keys):
l = left[key]
r = right[key]
if l.shape != r.shape:
print(f"[ERROR] step3lossfailure: shapestatusnotoneconsistent {key}")
return 1
diff = torch.abs(l.float() - r.float()).max().item()
if diff > max_diff:
max_diff = diff
if diff > args.tolerance:
print(f"[ERROR] step3lossfailure: Weightsdifferencedifferentexceedthresholdvalue {key} diff={diff:.2e}")
return 1
print(f"[OK] step3completed: max_diff={max_diff:.2e}")
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/step4_verify_quant_description.py
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
"""
VerificationProcessSteps4: VerificationQuantizationDescriptionfile
rootdataRules FileCheck quant_weight_description.json middleoflayerQuantizationtypetypewhether matchespreperiod.
"""
import os
import sys
import json
import argparse
from typing import List, Dict, Any
def load_json(path: str) -> Any:
with open(path, 'r', encoding='utf-8') as f:
return json.load(f)
def find_description_file(path: str) -> str:
"""infingerdefinePathsearchfindDescriptionfile"""
if os.path.isfile(path):
return path
p = os.path.join(path, "quant_weight_description.json")
if os.path.exists(p):
return p
return None
def verify_description(desc_path: str, rules_path: str) -> bool:
print("=" * 60)
print("Steps4: VerificationQuantizationDescriptionfile")
print("=" * 60)
# 1. searchfindandLoadDescriptionfile
real_desc_path = find_description_file(desc_path)
if not real_desc_path:
print(f"[ERROR] notfindtoQuantizationDescriptionfile (inPath: {desc_path})")
print(" periodexpectedfile: quant_weight_description.json or quant_model_description.json")
return False
print(f"[INFO] Descriptionfile: {real_desc_path}")
try:
desc_data = load_json(real_desc_path)
except Exception as e:
print(f"[ERROR] LoadDescriptionfilelossfailure: {e}")
return False
if not isinstance(desc_data, dict):
print(f"[ERROR] Descriptionfileformatformulaerrorerror: periodexpectedas JSON Object (dict)")
return False
# 2. LoadRules File
print(f"[INFO] Rules File: {rules_path}")
if not os.path.exists(rules_path):
print(f"[ERROR] Rules Filenotkeepin: {rules_path}")
return False
try:
rules = load_json(rules_path)
except Exception as e:
print(f"[ERROR] LoadRules Filelossfailure: {e}")
return False
if not isinstance(rules, list):
print(f"[ERROR] Rules Fileformatformulaerrorerror: periodexpectedas JSON Array (list)")
return False
# 3. Executecalibrateverify
print("\n[CHECK] openbeginmatchmatchRules...")
all_passed = True
total_checked_keys = 0
for i, rule in enumerate(rules):
quant_type = rule.get("quant_type")
keywords = rule.get("keywords", [])
if not quant_type or not keywords:
print(f"[WARNING] Rules #{i+1} formatformulanovalid (Missing quant_type or keywords), jumpexceed")
continue
print(f" > Rules #{i+1}: periodexpectedPackagecontain {keywords} ofWeightsas '{quant_type}'")
matched_keys = []
failed_keys = []
# iteratehistoryDescriptionfilemiddleofallhavekey
for key, value in desc_data.items():
# onlyCheckWeightsfile (throughoftenin order to .weight conclusiontail), avoidavoidCheck bias orotherotherattributeproperty
# If UserRulesinsideclearconfirmcomposecompletednotbandwidth .weight ofrelatedkeycharacter, thisinsidealsoCompatibility
if not isinstance(key, str):
continue
# Checkiswhethermatchmatchanyonerelatedkeycharacter
is_match = False
for kw in keywords:
if kw in key:
is_match = True
break
if is_match:
# silentrecognizeonlyCheck .weight conclusiontailofkey, dividenonRulesinsidedisplayformulaPackagecontain bias etc
# thisinsideascompletedthroughuseproperty, IpeoplefakesetuseuserProvidesof keyword footenoughtoolbody, orwhosilentrecognizeexceedfilternon weight
# modifyenterstrategystrategy: ifresult key Packagecontain keyword, thenperformCheck
# strictformatCheckvalue
if value != quant_type:
failed_keys.append((key, value))
else:
matched_keys.append(key)
total_checked_keys += len(matched_keys) + len(failed_keys)
if failed_keys:
all_passed = False
print(f" [FAILED] issueappear {len(failed_keys)} countnotmatchmatchitem (expandshowprevious10count):")
for k, v in failed_keys[:10]:
print(f" - {k}: actualactualvalue='{v}', periodexpectedvalue='{quant_type}'")
if len(failed_keys) > 10:
print(f" ... alsohave {len(failed_keys) - 10} count")
elif not matched_keys:
print(f" [WARNING] notfindtomatchmatchoughtRulesrelatedkeycharacterofanywhatWeightskey (canabilityisrelatedkeycharacterhaveerror?)")
else:
print(f" [OK] {len(matched_keys)} countWeightsitemVerificationthroughexceed")
print("-" * 60)
if all_passed and total_checked_keys > 0:
print(f"[SUCCESS] Verificationthroughexceed! allhavematchmatchitemaveragecharactermatchpreperiodQuantizationtypetype. ")
return True
elif total_checked_keys == 0:
print(f"[FAILED] Verificationlossfailure: notmatchmatchtoanywhatcharactermatchRulesofWeightsitem, pleaseCheckRulesrelatedkeycharacter. ")
return False
else:
print(f"[FAILED] Verificationlossfailure: keepinQuantizationtypetypenotmatchmatchofWeightsitem. ")
return False
def main():
parser = argparse.ArgumentParser(description="VerificationQuantizationDescriptionfileinsidecontent")
parser.add_argument("--desc-path", required=True, help="QuantizationOutputDirectoryorDescriptionfilePath")
parser.add_argument("--rules-path", required=True, help="calibrateverifyRulesJSONfilePath")
args = parser.parse_args()
success = verify_description(args.desc_path, args.rules_path)
sys.exit(0 if success else 1)
if __name__ == "__main__":
main()
SKILL.md
---
name: huawei-cloud-msmodelslim-model-adapt
description: |-
Create basic Transformers model adapters for msModelSlim. Implements required interfaces and completes a four-step verification workflow:
generate test model -> full fallback quantization -> weight verification -> quantization description validation. Use this skill when the user wants to: (1) create msModelSlim adapters for decoder-only LLM, (2) adapt understanding VLM text backbones for quantization, (3) implement W8A8/W4A16 quantization workflow for new models. Trigger: user mentions "msModelSlim", "adapter", "model adapter","quantization", "W8A8","W4A16", "transformers", "LLM", "VLM", "adapter creation", "适配器","模型适配", "量化", "模型适配器", "LLM量化"
compatibility:
- transformers >= 4.40.0
- msmodelslim >= 1.0.0
tags: [msModelSlim, adapter, quantization, model]
allowed-tools:
- python3
- bash
---
# Huawei Cloud msModelSlim Model Adapter
## Overview
This skill guides how to create basic adapters for new models to run
W8A8/W4A16 quantization workflows in msModelSlim.
**Architecture**: Model Analysis -> Adapter Creation -> Registration ->
Verification (4 Steps)
**Related Skills**:
- `huawei-cloud-msmodelslim-model-analysis` - Model structure analysis
before adapter implementation
- `huawei-cloud-ascend-profiler-db-explorer` - Optional: Performance
analysis after deployment
## Scope
**Supported**:
- Decoder-only LLM
- Understanding VLM (text/LLM backbone only)
**Not supported**:
- Multimodal generation (Stable Diffusion/Flux/Wan)
- Encoder-only models
- Non-Transformers architectures
## Architecture
```text
┌─────────────────────────────────────────────────────────────┐
│ msModelSlim Model Adapter Skill │
├─────────────────────────────────────────────────────────────┤
│ ┌──────────────────┐ ┌──────────────────────────────┐ │
│ │ Model Analysis │───▶│ Adapter Creation │ │
│ │ - config.json │ │ - LLM Adapter Template │ │
│ │ - modeling_*.py│ │ - VLM Adapter Template │ │
│ └──────────────────┘ │ - Required Interfaces │ │
│ └──────────────────────────────┘ │
│ │ │
│ ▼ │
│ ┌──────────────────┐ │
│ │ Registration │ │
│ │ & Installation │ │
│ └──────────────────┘ │
│ │ │
│ ▼ │
│ ┌──────────────────────────────────────────────────────┐ │
│ │ Verification (4 Steps) │ │
│ │ 1. Generate Test Model → 2. Full Fallback Quant │ │
│ │ 3. Weight Verification → 4. Quant Description │ │
│ └──────────────────────────────────────────────────────┘ │
└─────────────────────────────────────────────────────────────┘
```
## Architecture Components
This skill involves the following cloud services and components:
- **msModelSlim**: Huawei Cloud's model quantization framework for
efficient model compression
- **Transformers Library**: Hugging Face Transformers for model loading
and processing
- **ModelScope**: Model download and management platform
- **Ascend NPU**: Target hardware for quantized model deployment
## Use Cases
**Typical Problem Scenarios:**
- Need to deploy LLM models with reduced memory footprint on Ascend NPU
- Want to optimize inference speed without significant accuracy loss
- Migrating models that don't have built-in msModelSlim support
- Need W8A8/W4A16 quantization for decoder-only LLM or VLM text backbones
**Typical User Phrases:**
- "How to quantize my custom LLM model for Ascend?"
- "Create msModelSlim adapter for Qwen model"
- "Implement W4A16 quantization workflow"
- "Adapt my VLM text backbone for quantization"
- "How to add quantization support for new models?"
## Core Workflow
### 1. Preparation
- **Download Model**: Recommended to use `modelscope download` for
non-weight files.
- Example: `modelscope download --model <org>/<model> --local_dir
./models/<name> --exclude '*.safetensors'`
- **Analyze Model**: Read `config.json` and `modeling_*.py` to confirm
structure and implementation.
- See: [Model Analysis Guide](references/model_analysis.md)
### 2. Create Adapter
- **Use Templates**:
- LLM: `assets/model_adapter_template.py`
- VLM: `assets/vlm_model_adapter_template.py`
- **Implement Interfaces**: Implement `handle_dataset`, `init_model`,
`generate_model_visit`, `generate_model_forward`, `enable_kv_cache`.
- **Key Principles**:
- `visit` and `forward` must be strictly consistent.
- MoE models recommended to unpack to pure linear layers.
- See: [Implementation Guide](references/implementation_guide.md)
### 3. Registration & Installation
- Register model and entry in `config/config.ini`, then execute
`bash install.sh`.
- See: [Registration Guide](references/registration_guide.md)
### 4. Verify Adapter (Required)
- Must execute four-step verification: Generate test model -> Full
fallback quantization -> Verify full fallback model matches float
weights exactly and can load/save completely -> Verify actual
quantization workflow works (including description file rule
validation).
- See: [Verification Guide](references/verification_guide.md)
## Common Scripts
Scripts located in `scripts/` directory:
- `scripts/step1_generate_test_model.py`
- `scripts/step2_run_quantization.py`
- `scripts/step3_verify_weights.py`
- `scripts/step4_verify_quant_description.py`
## Prerequisites
### System Requirements
- Python 3.8+
- transformers >= 4.40.0
- msmodelslim >= 1.0.0
### Environment Check
> **Prerequisite check: Python3 + transformers + msmodelslim required**
>
> ```bash
> python3 --version # Python3 >= 3.8
> python3 -c "import transformers; print('OK')" # Transformers library
> python3 -c "import msmodelslim; print('OK')" # msModelSlim library
> ```
>
> If not installed: `pip3 install --user transformers msmodelslim`
## Reference Documents
| Document | Description |
| ---------- | ------------- |
| [Model Analysis Guide](references/model_analysis.md) | Model structure analysis guide |
| [Implementation Guide](references/implementation_guide.md) | Adapter implementation instructions |
| [Registration Guide](references/registration_guide.md) | Registration and installation guide |
| [Verification Guide](references/verification_guide.md) | Four-step verification workflow |
| [Interface Checklist](references/interface_checklist.md) | Required interface implementation checklist |
| [Core Workflow](references/core_workflow.md) | Core workflow documentation |
| [Acceptance Criteria](references/acceptance-criteria.md) | Functional acceptance criteria |
| [Troubleshooting](references/troubleshooting.md) | Common issues and solutions |
## Requirements
- transformers >= 4.40.0 installed
- msmodelslim >= 1.0.0 installed
- Transformers model to be adapted
- Understanding of target quantization scheme (W8A8/W4A16)
## Core Commands
```bash
# Create model adapter
python3 scripts/create_adapter.py \
--model Qwen2-7B \
--quantization W8A8
# Run four-step verification
python3 scripts/verify_adapter.py --adapter ./adapter.py
```
## Parameter Confirmation
| Parameter | Description | Required |
| ---------- | ------------- | ---------- |
| model | Model name or path | Yes |
| quantization | Quantization scheme (W8A8/W4A16) | Yes |
| output | Adapter output path | No |