GPT2 model (Regression)

This notebook contains some example of how to use the GPT2-based models in this NLP library

In this series, we walk through some of the capability of this library: single-head classification, multi-head classification, multi-label classification, and regression. If you want a more detailed tutorial, check this out

import os

#This will specify a (or a list) of GPUs for training
os.environ['CUDA_VISIBLE_DEVICES'] = "0"

from that_nlp_library.text_transformation import *
from that_nlp_library.text_augmentation import *
from that_nlp_library.text_main import *
from that_nlp_library.utils import seed_everything

from underthesea import text_normalize
from functools import partial
from pathlib import Path
import pandas as pd
import numpy as np
import nlpaug.augmenter.char as nac
from datasets import load_dataset
import random
from transformers import AutoTokenizer
from datasets import Dataset
import torch
from transformers.models.gpt2.modeling_gpt2 import GPT2Model

Define the custom augmentation function

def nlp_aug_stochastic(x,aug=None,p=0.5):
    if not isinstance(x,list): 
        if random.random()<p: return aug.augment(x)[0]
        return x
    news=[]
    originals=[]
    for _x in x:
        if random.random()<p: news.append(_x)
        else: originals.append(_x)
    # only perform augmentation when needed
    if len(news): news = aug.augment(news)
    return news+originals

aug = nac.KeyboardAug(aug_char_max=3,aug_char_p=0.1,aug_word_p=0.07)
nearby_aug_func = partial(nlp_aug_stochastic,aug=aug,p=0.3)

Create a TextDataController object

We will reuse the data and the preprocessings in this tutorial

dset = load_dataset('sample_data',data_files=['Womens_Clothing_Reviews.csv'],split='train')

tdc = TextDataController(dset,
                         main_text='Review Text',
                         label_names='Rating',
                         sup_types='regression',
                         filter_dict={'Review Text': lambda x: x is not None},
                         metadatas=['Title','Division Name'],
                         content_transformations=[text_normalize,str.lower],
                         content_augmentations= [nearby_aug_func,str.lower], 
                         # add "str.lower" here because nearby_aug might return uppercase character
                         val_ratio=0.2,
                         batch_size=1000,
                         seed=42,
                         num_proc=20,
                         verbose=False
                        )

Define our tokenizer for GPT2

_tokenizer = AutoTokenizer.from_pretrained('gpt2')

/home/quan/anaconda3/envs/nlp_dev/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.
  warnings.warn(

_tokenizer.pad_token = _tokenizer.eos_token
_tokenizer.padding_side = 'left'

print(_tokenizer)
print(len(_tokenizer))

GPT2TokenizerFast(name_or_path='gpt2', vocab_size=50257, model_max_length=1024, is_fast=True, padding_side='left', truncation_side='right', special_tokens={'bos_token': '<|endoftext|>', 'eos_token': '<|endoftext|>', 'unk_token': '<|endoftext|>', 'pad_token': '<|endoftext|>'}, clean_up_tokenization_spaces=True),  added_tokens_decoder={
    50256: AddedToken("<|endoftext|>", rstrip=False, lstrip=False, single_word=False, normalized=True, special=True),
}
50257

Process and tokenize our dataset

tdc.process_and_tokenize(_tokenizer,max_length=100,shuffle_trn=True)

tdc.main_ddict

DatasetDict({
    train: Dataset({
        features: ['Title', 'Review Text', 'Rating', 'Division Name', 'label', 'input_ids', 'attention_mask'],
        num_rows: 18112
    })
    validation: Dataset({
        features: ['Title', 'Review Text', 'Rating', 'Division Name', 'label', 'input_ids', 'attention_mask'],
        num_rows: 4529
    })
})

Model Experiment: GPT2 Single-Head Regression

from that_nlp_library.models.roberta.classifiers import ConcatHeadSimple
from that_nlp_library.models.gpt2.classifiers import *
from that_nlp_library.model_main import *
from sklearn.metrics import mean_absolute_error,mean_squared_log_error,f1_score, accuracy_score

Using HuggingFace model initialization

from transformers.models.gpt2.modeling_gpt2 import GPT2ForSequenceClassification

num_classes=1

seed_everything(42)
model = GPT2ForSequenceClassification.from_pretrained('gpt2',num_labels=num_classes)
model = model.to('cuda:0')

Some weights of GPT2ForSequenceClassification were not initialized from the model checkpoint at gpt2 and are newly initialized: ['score.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

model.config.pad_token_id = model.config.eos_token_id
model.resize_token_embeddings(len(_tokenizer))

Embedding(50257, 768)

metric_funcs = [mean_absolute_error,mean_squared_log_error]
controller = ModelController(model,tdc,seed=42)

lr = 1e-4
bs=32
wd=0.01
epochs= 3

controller.fit(epochs,lr,
               metric_funcs=metric_funcs,
               batch_size=bs,
               weight_decay=wd,
               save_checkpoint=False,
               compute_metrics=compute_metrics,
              )

[849/849 07:25, Epoch 3/3]

Epoch	Training Loss	Validation Loss	Mean Absolute Error Rating	Mean Squared Log Error Rating
1	No log	0.401165	0.452420	0.026643
2	0.837500	0.346818	0.407272	0.022015
3	0.837500	0.335389	0.400436	0.021186

Using GPT2-based model (no concatenation), but with a custom head to limit the output range

num_classes=1

gpt2_body = GPT2Model.from_pretrained('gpt2')

/home/quan/anaconda3/envs/nlp_dev/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.
  warnings.warn(

class GPT2SigmoidRange(torch.nn.Module):
    def __init__(self,
                 config,
                 high,
                 low,
                 **kwargs
                ):
        super().__init__()
        self.high=high
        self.low=low
        self.score = torch.nn.Linear(config.n_embd, config.num_labels, bias=False)
    def forward(self, inp, **kwargs):
        logits = self.score(inp)
        return torch.sigmoid(logits)*(self.high-self.low)+self.low

_model_kwargs={
    # overall model hyperparams
    'head_class_sizes':num_classes,
    'head_class': GPT2SigmoidRange,
    # classfication head hyperparams
    'high':5, # the maximum rating
    'low': 1, # the minimum rating
}

model = model_init_classification(model_class = GPT2BaseForSequenceClassification,
                                  cpoint_path = 'gpt2', 
                                  output_hidden_states=False,
                                  seed=42,
                                  body_model=gpt2_body,
                                  model_kwargs = _model_kwargs)

Loading body weights. This assumes the body is the very first block of your custom architecture
Total parameters: 124440576
Total trainable parameters: 124440576

model.body_model.resize_token_embeddings(len(_tokenizer))

Embedding(50257, 768)

metric_funcs = [mean_absolute_error,mean_squared_log_error]
controller = ModelController(model,tdc,seed=42)

seed_everything(42)

lr = 1e-4
bs=32
wd=0.01
epochs= 3

controller.fit(epochs,lr,
               metric_funcs=metric_funcs,
               batch_size=bs,
               weight_decay=wd,
               save_checkpoint=False,
               compute_metrics=compute_metrics,
              )

[849/849 07:49, Epoch 3/3]

Epoch	Training Loss	Validation Loss	Mean Absolute Error Rating	Mean Squared Log Error Rating
1	No log	0.392251	0.473784	0.024639
2	0.591400	0.348463	0.412898	0.021849
3	0.591400	0.326785	0.391602	0.020824

Using the GPT2 custom model (concatenation)

num_classes=1

gpt2_body = GPT2Model.from_pretrained('gpt2')

/home/quan/anaconda3/envs/nlp_dev/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.
  warnings.warn(

_model_kwargs={
    # overall model hyperparams
    'head_class_sizes':num_classes,
    'head_class': ConcatHeadSimple,
    # classfication head hyperparams
    'layer2concat':4,
    'classifier_dropout':0.1 
}

model = model_init_classification(model_class = GPT2HiddenStateConcatForSequenceClassification,
                                  cpoint_path = 'gpt2', 
                                  output_hidden_states=True, # since we are using 'hidden layer contatenation' technique
                                  seed=42,
                                  body_model=gpt2_body,
                                  model_kwargs = _model_kwargs)
model.body_model.resize_token_embeddings(len(_tokenizer))

Loading body weights. This assumes the body is the very first block of your custom architecture
Total parameters: 124442881
Total trainable parameters: 124442881

Embedding(50257, 768)

metric_funcs = [mean_absolute_error,mean_squared_log_error]
controller = ModelController(model,tdc,seed=42)

seed_everything(42)

lr = 1e-4
bs=32
wd=0.01
epochs= 3

controller.fit(epochs,lr,
               metric_funcs=metric_funcs,
               batch_size=bs,
               weight_decay=wd,
               save_checkpoint=False,
               compute_metrics=compute_metrics,
              )

[849/849 06:02, Epoch 3/3]

Epoch	Training Loss	Validation Loss	Mean Absolute Error Rating	Mean Squared Log Error Rating
1	No log	0.590046	0.550116	0.037543
2	2.443300	0.409290	0.443936	0.025740
3	2.443300	0.379117	0.441241	0.023792

controller.trainer.model.save_pretrained('./sample_weights/my_model')

Predict Validation

_model_kwargs

{'head_class_sizes': 1,
 'head_class': that_nlp_library.models.roberta.classifiers.ConcatHeadSimple,
 'layer2concat': 4,
 'classifier_dropout': 0.1}

trained_model = model_init_classification(model_class = GPT2HiddenStateConcatForSequenceClassification,
                                          cpoint_path = Path('./sample_weights/my_model'), 
                                          output_hidden_states=True,
                                          seed=42,
                                          model_kwargs = _model_kwargs)

controller = ModelController(trained_model,tdc,seed=42)

Total parameters: 124442881
Total trainable parameters: 124442881

df_val = controller.predict_ddict(ds_type='validation')

-------------------- Start making predictions --------------------

df_val = df_val.to_pandas()
df_val.head()

	Title	Review Text	Rating	Division Name	label	input_ids	attention_mask	pred_Rating
0		general . . this picture doesn't do the skirt ...	5.0	general	5.0	[50256, 50256, 50256, 50256, 50256, 50256, 502...	[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...	4.741378
1		general . . easy to wear ! cute , comfy ... wi...	4.0	general	4.0	[50256, 50256, 50256, 50256, 50256, 50256, 502...	[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...	4.796369
2		general . . nice sweater , just did not look g...	3.0	general	3.0	[50256, 50256, 50256, 50256, 50256, 50256, 502...	[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...	2.616789
3	nice cropped jacket	general . nice cropped jacket . this jacket wa...	5.0	general	5.0	[50256, 50256, 50256, 50256, 50256, 50256, 502...	[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...	4.419146
4	great dress!	general petite . great dress ! . i wasn't plan...	5.0	general petite	5.0	[24622, 4273, 578, 764, 1049, 6576, 5145, 764,...	[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...	5.079652

You can try to get your metric to see if it matches your last traing epoch’s above

mean_absolute_error(df_val['label'],df_val['pred_Rating'])

0.441279638942753

mean_squared_log_error(df_val['label'],df_val['pred_Rating'])

0.023789442640954332

Predict Test set

We will go through details on how to make a prediction on a completely new and raw dataset using our trained model. For now, let’s reuse the sample csv and pretend it’s our test set

df_test = pd.read_csv('sample_data/Womens_Clothing_Reviews.csv',encoding='utf-8-sig').sample(frac=0.2,random_state=1)


# save the label, as we will calculate some metrics later
true_labels = df_test[~df_test['Review Text'].isna()].Rating.values 

# drop the label (you don't need to, but this is necessary to simulate an actual test set)
df_test.drop('Rating',axis=1,inplace=True)

_test_dset = Dataset.from_pandas(df_test)
_test_dset_predicted = controller.predict_raw_dset(_test_dset,
                                                   do_filtering=True, # since we have some text filtering in the processing
                                                  )

-------------------- Start making predictions --------------------

df_test_predicted = _test_dset_predicted.to_pandas()

df_test_predicted.head()

	Title	Review Text	Division Name	input_ids	attention_mask	pred_Rating
0	perfect for work and play	general . perfect for work and play . this shi...	general	[50256, 50256, 50256, 50256, 50256, 50256, 502...	[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...	4.962859
1		general petite . . i don't know why i had the ...	general petite	[24622, 4273, 578, 764, 764, 1312, 836, 470, 7...	[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...	4.025272
2	great pants	general petite . great pants . thes e cords ar...	general petite	[50256, 50256, 50256, 50256, 50256, 50256, 502...	[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...	4.888588
3	surprisingly comfy for a button down	general petite . surprisingly comfy for a butt...	general petite	[24622, 4273, 578, 764, 12362, 401, 24928, 329...	[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...	4.510598
4	short and small	general petite . short and small . the shirt i...	general petite	[50256, 50256, 50256, 50256, 50256, 50256, 502...	[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...	2.427316

Let’s quickly check the score to make sure everything works correctly

mean_absolute_error(true_labels,df_test_predicted['pred_Rating'])

0.41902880738461595

mean_squared_log_error(true_labels,df_test_predicted['pred_Rating'])

0.022995927657672072

raw_content={'Review Text': 'This shirt is so comfortable I love it!',
             'Title': 'Great shirt',
             'Division Name': 'general'}
raw_content

{'Review Text': 'This shirt is so comfortable I love it!',
 'Title': 'Great shirt',
 'Division Name': 'general'}

df_result = controller.predict_raw_text(raw_content)

-------------------- Start making predictions --------------------

df_result

{'Review Text': ['general . great shirt . this shirt is so comfortable i love it !'],
 'Title': ['great shirt'],
 'Division Name': ['general'],
 'input_ids': [[24622,
   764,
   1049,
   10147,
   764,
   428,
   10147,
   318,
   523,
   6792,
   1312,
   1842,
   340,
   5145]],
 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]],
 'pred_Rating': [4.925951957702637]}

Model Experiment: GPT2 Multi-Head Regression

from that_nlp_library.models.roberta.classifiers import ConcatHeadSimple
from that_nlp_library.models.gpt2.classifiers import *
from that_nlp_library.model_main import *
from sklearn.metrics import mean_absolute_error,mean_squared_log_error,f1_score, accuracy_score

Re-define the TextDataController

dset = load_dataset('sample_data',data_files=['Womens_Clothing_Reviews.csv'],split='train')

tdc = TextDataController(dset,
                         main_text='Review Text',
                         label_names=['Rating','Department Name'],
                         sup_types=['regression','classification'],
                         filter_dict={'Review Text': lambda x: x is not None,
                                      'Department Name': lambda x: x is not None},
                         metadatas=['Title'],
                         content_transformations=[text_normalize,str.lower],
                         content_augmentations= [nearby_aug_func,str.lower], 
                         val_ratio=0.2,
                         batch_size=1000,
                         seed=42,
                         num_proc=20,
                         verbose=False
                        )

_tokenizer = AutoTokenizer.from_pretrained('gpt2')

_tokenizer.pad_token = _tokenizer.eos_token
_tokenizer.padding_side = 'left'