使用 GPTQ、AWQ 和 Bitsandbytes 进行模型量化

pip install transformers optimum accelerate auto-gptq
python ./quant_autogptq.py meta-llama/Llama-2-7b-chat-hf gptq_checkpoints c4 --bits 4 --group_size 128 --desc_act 1 --damp 0.1 --seqlen 4096

from datasets import load_dataset
import os
import random
import torch
import numpy as np
from auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfig


model_id = "meta-llama/Llama-2-7b-hf"


tokenizer = AutoTokenizer.from_pretrained(model_id, 
                                               use_fast=True,)
def get_c4():
 
    traindata = load_dataset(
        'allenai/c4', data_files={'train': 'en/c4-train.00000-of-01024.json.gz'}, split='train', use_auth_token=False
    )
    seqlen  = 4096
    trainloader = []
    for _ in range(self.num_samples):
        while True:
            i = random.randint(0, len(traindata) - 1)
            trainenc = tokenizer(traindata[i]['text'], return_tensors='pt')
            if trainenc.input_ids.shape[1] >= self.seqlen:
                break
        i = random.randint(0, trainenc.input_ids.shape[1] - self.seqlen - 1)
        j = i + seqlen
        inp = trainenc.input_ids[:, i:j]
        attention_mask = torch.ones_like(inp)
        trainloader.append({'input_ids':inp,'attention_mask': attention_mask})
    return trainloader

quantize_config = BaseQuantizeConfig(
    bits=bits,
    group_size=group_size,
    desc_act=desc_act,
    damp_percent=damp
)


torch_dtype = torch.float16


traindataset = get_c4()


model = AutoGPTQForCausalLM.from_pretrained(model_id, quantize_config=quantize_config,
                                            low_cpu_mem_usage=True, torch_dtype=torch_dtype)




model.quantize(traindataset, use_triton=self.use_triton, batch_size=self.batch_size, cache_examples_on_gpu=self.cache_examples)

output_dir = 'Llama-2-7b-hf-gptq'


model.save_quantized(output_dir, use_safetensors=True)
tokenizer.save_pretrained(output_dir)

from huggingface_hub import HfApi


api = HfApi()
api.upload_folder(
    folder_path="Llama-2-7b-hf-gptq",
    repo_id="luv2261/Llama-2-7b-hf-gptq",
    repo_type="model",
)

from transformers import AutoTokenizer, AutoModelForCausalLM


model_id = "luv2261/Llama-2-7b-hf-gptq"


model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto")
tokenizer = AutoTokenizer.from_pretrained(model_id)


prompt = "<s>[INST] Write a tweet on future of AI [/INST]"
inputs = tokenizer(prompt, return_tensors="pt").to(0)


out = model.generate(**inputs, max_new_tokens=250, temperature = 0.6, top_p=0.95, tok_k=40)


print(tokenizer.decode(out[0], skip_special_tokens=True))

from awq import AutoAWQForCausalLM
from transformers import AutoTokenizer


model_path = 'meta-llama/Llama-2-7b-hf'


quant_config = { "zero_point": True, "q_group_size": 128, "w_bit": 4, "version": "GEMM" }
# Load model
model = AutoAWQForCausalLM.from_pretrained(
    model_path, **{"low_cpu_mem_usage": True, "use_cache": False}
)
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
# Quantize
model.quantize(tokenizer, quant_config=quant_config)

from transformers import AwqConfig, AutoConfig
from huggingface_hub import HfApi


# modify the config file so that it is compatible with transformers integration
quantization_config = AwqConfig(
    bits=quant_config["w_bit"],
    group_size=quant_config["q_group_size"],
    zero_point=quant_config["zero_point"],
    version=quant_config["version"].lower(),
).to_dict()


# the pretrained transformers model is stored in the model attribute + we need to pass a dict
model.model.config.quantization_config = quantization_config
# a second solution would be to use Autoconfig and push to hub (what we do at llm-awq)


# Save quantized model
quant_path = 'Llama-2-7b-hf-awq'


model.save_quantized(quant_path)
tokenizer.save_pretrained(quant_path)
print(f'Model is quantized and saved at "{quant_path}"')

from transformers import BitsAndBytesConfig
from torch import bfloat16


# Our 4-bit configuration to load the LLM with less GPU memory
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,  # 4-bit quantization
    bnb_4bit_quant_type='nf4',  # Normalized float 4
    bnb_4bit_use_double_quant=True,  # Second quantization after the first
    bnb_4bit_compute_dtype=bfloat16  # Computation type
)

from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline


# Zephyr with BitsAndBytes Configuration
tokenizer = AutoTokenizer.from_pretrained("HuggingFaceH4/zephyr-7b-alpha")


model = AutoModelForCausalLM.from_pretrained(
    "HuggingFaceH4/zephyr-7b-alpha",
    quantization_config=bnb_config,
    device_map='auto',
)


# Create a pipeline
pipe = pipeline(model=model, tokenizer=tokenizer, task='text-generation')


prompt = '''
<|system|> \
You are a friendly chatbot who always responds in the style of a pirate.\
</s>\
<|user|>\
Write a tweet on future of AI. </s>\
<|assistant|>
'''


outputs = pipe(
    prompt, 
    max_new_tokens=256, 
    do_sample=True, 
    temperature=0.7, 
    top_p=0.95
)
print(outputs[0]["generated_text"])