checkpoint configuration file
Download into your container the following github repository which contains the configuration scripts and the execution scripts for the convert_checkpoint function
After having used the helper scripts, you can now run the checkpoint.py command
git clone https://github.com/Continuum-Labs-HQ/tensorrt-continuum.gitThis will download the following files:
Checkpoint Conversation Configuration YAML File
A run_convert_checkpoint.py script that you execute when you are comfortable with the configuration
YAML Configuration File
Enter in your arguments based on the configuration files from the Huggingface files - they have to be consistent
model:
model_dir: ./llama-2-7b-chat-hf
output_dir: ../llama-2-7b-chat-hf-output
dtype: float16 # Choices: float32, bfloat16, float16
# Suggestions:
# - Use float16 for better performance with minimal accuracy loss
# - Use bfloat16 for a balance between performance and accuracy
# - Use float32 for maximum accuracy but slower performance
checkpoint:
tp_size: 1 # Tensor parallelism size
pp_size: 1 # Pipeline parallelism size
# Suggestions:
# - Increase tp_size and pp_size for distributed training across multiple GPUs
# - Keep tp_size and pp_size as 1 for single GPU training
vocab_size: 32000
# Suggestions:
# - Adjust vocab_size based on the specific tokenizer and model requirements
n_positions: 2048
# Suggestions:
# - Increase n_positions for longer sequence lengths
# - Decrease n_positions for shorter sequence lengths to save memory
n_layer: 32
# Suggestions:
# - Adjust n_layer based on the desired model depth
# - Increase n_layer for more complex models
# - Decrease n_layer for simpler models or faster training
n_head: 32
# Suggestions:
# - Adjust n_head based on the desired number of attention heads
# - Increase n_head for more fine-grained attention
# - Decrease n_head for faster training or smaller models
n_embd: 4096
# Suggestions:
# - Adjust n_embd based on the desired hidden size
# - Increase n_embd for larger models with more capacity
# - Decrease n_embd for smaller models or faster training
inter_size: 11008
# Suggestions:
# - Adjust inter_size based on the desired intermediate size in the feed-forward layers
# - Increase inter_size for more capacity in the feed-forward layers
# - Decrease inter_size for smaller models or faster training
# Additional checkpoint arguments
meta_ckpt_dir: null # ./path/to/meta/checkpoint
n_kv_head: null # 32
rms_norm_eps: 1e-6
use_weight_only: false
disable_weight_only_quant_plugin: false
weight_only_precision: int8 # Choices: int8, int4, int4_gptq
smoothquant: null # 0.5
per_channel: false
per_token: false
int8_kv_cache: false
ammo_quant_ckpt_path: null # ./path/to/ammo/quant/checkpoint
per_group: false
load_by_shard: false
hidden_act: silu
rotary_base: 10000.0
group_size: 128
dataset_cache_dir: null # ./path/to/dataset/cache
load_model_on_cpu: false
use_parallel_embedding: false
embedding_sharding_dim: 0 # Choices: 0, 1
use_embedding_sharing: false
workers: 1
moe_num_experts: 0
moe_top_k: 0
moe_tp_mode: 0
moe_renorm_mode: 1
save_config_only: falsemodel:
model_dir: ./llama-2-7b-chat-hf
output_dir: ../llama-2-7b-chat-hf-output
dtype: float16 # Choices: float32, bfloat16, float16
# Suggestions:
# - Use float16 for better performance with minimal accuracy loss
# - Use bfloat16 for a balance between performance and accuracy
# - Use float32 for maximum accuracy but slower performance
checkpoint:
tp_size: 1 # Tensor parallelism size
pp_size: 1 # Pipeline parallelism size
# Suggestions:
# - Increase tp_size and pp_size for distributed training across multiple GPUs
# - Keep tp_size and pp_size as 1 for single GPU training
vocab_size: 32000
# Suggestions:
# - Adjust vocab_size based on the specific tokenizer and model requirements
n_positions: 2048
# Suggestions:
# - Increase n_positions for longer sequence lengths
# - Decrease n_positions for shorter sequence lengths to save memory
n_layer: 32
# Suggestions:
# - Adjust n_layer based on the desired model depth
# - Increase n_layer for more complex models
# - Decrease n_layer for simpler models or faster training
n_head: 32
# Suggestions:
# - Adjust n_head based on the desired number of attention heads
# - Increase n_head for more fine-grained attention
# - Decrease n_head for faster training or smaller models
n_embd: 4096
# Suggestions:
# - Adjust n_embd based on the desired hidden size
# - Increase n_embd for larger models with more capacity
# - Decrease n_embd for smaller models or faster training
inter_size: 11008
# Suggestions:
# - Adjust inter_size based on the desired intermediate size in the feed-forward layers
# - Increase inter_size for more capacity in the feed-forward layers
# - Decrease inter_size for smaller models or faster training
# Additional checkpoint arguments
meta_ckpt_dir: null # ./path/to/meta/checkpoint
n_kv_head: null # 32
rms_norm_eps: 1e-6
use_weight_only: false
disable_weight_only_quant_plugin: false
weight_only_precision: int8 # Choices: int8, int4, int4_gptq
smoothquant: null # 0.5
per_channel: false
per_token: false
int8_kv_cache: false
ammo_quant_ckpt_path: null # ./path/to/ammo/quant/checkpoint
per_group: false
load_by_shard: false
hidden_act: silu
rotary_base: 10000.0
group_size: 128
dataset_cache_dir: null # ./path/to/dataset/cache
load_model_on_cpu: false
use_parallel_embedding: false
embedding_sharding_dim: 0 # Choices: 0, 1
use_embedding_sharing: false
workers: 1
moe_num_experts: 0
moe_top_k: 0
moe_tp_mode: 0
moe_renorm_mode: 1
save_config_only: falseLast updated
Was this helpful?