Roberta model with Conditional Probability

This notebook contains some example of how to use the Roberta-based models in this NLP library

In this tutorial, we walk through a special case of classification with multiple heads. This is inspired by this paper: https://arxiv.org/pdf/1911.06475.pdf

import os

#This will specify a (or a list) of GPUs for training
os.environ['CUDA_VISIBLE_DEVICES'] = "0"

from that_nlp_library.text_transformation import *
from that_nlp_library.text_augmentation import *
from that_nlp_library.text_main import *
from that_nlp_library.utils import seed_everything

from underthesea import text_normalize
from functools import partial
from pathlib import Path
import pandas as pd
import numpy as np
import nlpaug.augmenter.char as nac
from datasets import load_dataset
import random
from transformers import RobertaTokenizer
from datasets import Dataset

Define the custom augmentation function

def nlp_aug_stochastic(x,aug=None,p=0.5):
    if not isinstance(x,list): 
        if random.random()<p: return aug.augment(x)[0]
        return x
    news=[]
    originals=[]
    for _x in x:
        if random.random()<p: news.append(_x)
        else: originals.append(_x)
    # only perform augmentation when needed
    if len(news): news = aug.augment(news)
    return news+originals

aug = nac.KeyboardAug(aug_char_max=3,aug_char_p=0.1,aug_word_p=0.07)
nearby_aug_func = partial(nlp_aug_stochastic,aug=aug,p=0.3)

Create a TextDataController object

We will reuse the data and the preprocessings in this tutorial

dset = load_dataset('sample_data',data_files=['Womens_Clothing_Reviews.csv'],split='train')

tdc = TextDataController(dset,
                         main_text='Review Text',
                         label_names=['Division Name','Department Name'],
                         sup_types=['classification','classification'],
                         filter_dict={'Review Text': lambda x: x is not None,
                                      'Department Name': lambda x: x is not None,
                                     },
                         metadatas=['Title'],
                         content_transformations=[text_normalize,str.lower],
                         content_augmentations= [nearby_aug_func,str.lower], 
                         val_ratio=0.2,
                         batch_size=1000,
                         seed=42,
                         num_proc=20,
                         verbose=False
                        )

Define our tokenizer for Roberta

_tokenizer = RobertaTokenizer.from_pretrained('roberta-base')

/home/quan/anaconda3/envs/nlp_dev/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.
  warnings.warn(

Process and tokenize our dataset

tdc.process_and_tokenize(_tokenizer,max_length=100,shuffle_trn=True)

tdc.main_ddict

DatasetDict({
    train: Dataset({
        features: ['Title', 'Review Text', 'Division Name', 'Department Name', 'label', 'input_ids', 'attention_mask'],
        num_rows: 18101
    })
    validation: Dataset({
        features: ['Title', 'Review Text', 'Division Name', 'Department Name', 'label', 'input_ids', 'attention_mask'],
        num_rows: 4526
    })
})

tdc.main_ddict['validation']['label'][:5]

[[1, 2], [1, 4], [0, 4], [1, 1], [1, 1]]

Model Experiment: Roberta Multi-Head Classification using Conditional Probability (with Hidden Layer Concatenation)

from that_nlp_library.models.roberta.classifiers import *
from that_nlp_library.model_main import *
from sklearn.metrics import f1_score, accuracy_score
from that_nlp_library.models.roberta.conditional_prob_classifiers import *
from transformers.models.roberta.modeling_roberta import RobertaModel
import torch

Build Conditional Mask

tdc.label_names

['Division Name', 'Department Name']

tdc.label_lists

[['General', 'General Petite', 'Initmates'],
 ['Bottoms', 'Dresses', 'Intimate', 'Jackets', 'Tops', 'Trend']]

df_trn = tdc.main_ddict['train'].to_pandas()

df_labels = pd.DataFrame(df_trn['label'].tolist())
df_labels.columns=tdc.label_names

df_labels.head()

	Division Name	Department Name
0	0	4
1	1	1
2	1	1
3	1	3
4	0	1

standard_mask = build_standard_condition_mask(df_labels,*tdc.label_names)

standard_mask

tensor([[ True, False, False,  True,  True, False,  True,  True,  True],
        [False,  True, False,  True,  True,  True,  True,  True,  True],
        [False, False,  True, False, False,  True, False, False, False]])

Explain the first row of the mask

standard_mask[0]

tensor([ True, False, False,  True,  True, False,  True,  True,  True])

Slicing the first portion for Division Name (the first 3 values), show string for True mask

for i in torch.where(standard_mask[0][:len(tdc.label_lists[0])]==True)[0]:
    print(tdc.label_lists[0][i])

General

Slicing the first portion for Department Name, show string for True mask. The results are the sub-category of Division Name

for i in torch.where(standard_mask[0][len(tdc.label_lists[0]):]==True)[0]:
    print(tdc.label_lists[1][i])

Bottoms
Dresses
Jackets
Tops
Trend

# let's double check with the original data
np.sort(df_trn[df_trn['Division Name']=='General']['Department Name'].unique())

array(['Bottoms', 'Dresses', 'Jackets', 'Tops', 'Trend'], dtype=object)

Define and train a custom Roberta model

roberta_body = RobertaModel.from_pretrained('roberta-base')

/home/quan/anaconda3/envs/nlp_dev/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.
  warnings.warn(
Some weights of RobertaModel were not initialized from the model checkpoint at roberta-base and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

_model_kwargs={
    # overall model hyperparams
    'size_l1':len(tdc.label_lists[0]),
    'size_l2':len(tdc.label_lists[1]),
    'standard_mask':standard_mask,
    'layer2concat':2,
    'head_class': ConcatHeadSimple,
    # classfication head hyperparams
    'classifier_dropout':0.1 
}

model = model_init_classification(model_class = RobertaHSCCProbSequenceClassification,
                                  cpoint_path = 'roberta-base', 
                                  output_hidden_states=True,
                                  seed=42,
                                  body_model=roberta_body,
                                  model_kwargs = _model_kwargs)

metric_funcs = [partial(f1_score,average='macro'),accuracy_score]
controller = ModelController(model,tdc,seed=42)

Loading body weights. This assumes the body is the very first block of your custom architecture
Total parameters: 124659465
Total trainable parameters: 124659465

And we can start training our model

seed_everything(42)

lr = 1e-4
bs=32
wd=0.01
epochs= 3

controller.fit(epochs,lr,
               metric_funcs=metric_funcs,
               batch_size=bs,
               weight_decay=wd,
               save_checkpoint=False,
               compute_metrics=compute_metrics,
              )

[849/849 05:25, Epoch 3/3]

Epoch	Training Loss	Validation Loss	F1 Score Division name	Accuracy Score Division name	F1 Score Department name	Accuracy Score Department name
1	No log	0.109697	0.419514	0.615113	0.650357	0.868979
2	0.141400	0.096176	0.451226	0.613566	0.682490	0.881131
3	0.141400	0.094754	0.447835	0.614229	0.682274	0.883120

controller.trainer.model.save_pretrained('./sample_weights/my_model')

Make predictions

Load trained model

_model_kwargs

{'size_l1': 3,
 'size_l2': 6,
 'standard_mask': tensor([[ True, False, False,  True,  True, False,  True,  True,  True],
         [False,  True, False,  True,  True,  True,  True,  True,  True],
         [False, False,  True, False, False,  True, False, False, False]]),
 'layer2concat': 2,
 'head_class': that_nlp_library.models.roberta.classifiers.ConcatHeadSimple,
 'classifier_dropout': 0.1}

trained_model = model_init_classification(model_class = RobertaHSCCProbSequenceClassification,
                                          cpoint_path = Path('./sample_weights/my_model'), 
                                          output_hidden_states=True,
                                          seed=42,
                                          model_kwargs = _model_kwargs)

controller = ModelController(trained_model,tdc,seed=42)

Some weights of the model checkpoint at sample_weights/my_model were not used when initializing RobertaHSCCProbSequenceClassification: ['body_model.pooler.dense.bias', 'body_model.pooler.dense.weight']
- This IS expected if you are initializing RobertaHSCCProbSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing RobertaHSCCProbSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).

Total parameters: 124068873
Total trainable parameters: 124068873

Predict Train/Validation set

df_val = controller.predict_ddict(ds_type='validation')

-------------------- Start making predictions --------------------

df_val = df_val.to_pandas()
df_val.head()

	Title	Review Text	Division Name	Department Name	label	input_ids	attention_mask	pred_Division Name	pred_prob_Division Name	pred_Department Name	pred_prob_Department Name
0		. such a fun jacket ! great to wear in the spr...	General Petite	Intimate	[1, 2]	[0, 4, 215, 10, 1531, 8443, 27785, 372, 7, 356...	[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...	General	0.591014	Jackets	0.898804
1	simple and elegant	simple and elegant . i thought this shirt was ...	General Petite	Tops	[1, 4]	[0, 41918, 8, 14878, 479, 939, 802, 42, 6399, ...	[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...	General	0.533907	Tops	0.999752
2	retro and pretty	retro and pretty . this top has a bit of a ret...	General	Tops	[0, 4]	[0, 4903, 1001, 8, 1256, 479, 42, 299, 34, 10,...	[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...	General	0.564118	Tops	0.999757
3	summer/fall wear	summer / fall wear . i first spotted this on a...	General Petite	Dresses	[1, 1]	[0, 18581, 2089, 1589, 1136, 3568, 479, 939, 7...	[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...	General	0.520808	Dresses	0.999089
4	perfect except slip	perfect except slip . this is my new favorite ...	General Petite	Dresses	[1, 1]	[0, 20473, 4682, 9215, 479, 42, 16, 127, 92, 2...	[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...	General	0.559546	Dresses	0.999006

You can try to get your metric to see if it matches your last traing epoch’s above

f1_score(df_val['Division Name'],df_val['pred_Division Name'],average='macro')
# 0.4486166839108015

0.4479421567807432

f1_score(df_val['Department Name'],df_val['pred_Department Name'],average='macro')
# 0.6818255247330748

0.6822742871946978

Predict Test set

We will go through details on how to make a prediction on a completely new and raw dataset using our trained model. For now, let’s reuse the sample csv and pretend it’s our test set

df_test = pd.read_csv('sample_data/Womens_Clothing_Reviews.csv',encoding='utf-8-sig').sample(frac=0.2,random_state=1)
# drop NaN values in the label column
df_test = df_test[~df_test['Department Name'].isna()].reset_index(drop=True)

# save the label, as we will calculate some metrics later. We also filter out labels with NaN Review Text,
# as there will be a filtering processing on the test set
true_labels = df_test.loc[~df_test['Review Text'].isna(),'Department Name'].values 

# drop the label (you don't need to, but this is necessary to simulate an actual test set)
df_test.drop(['Division Name','Department Name'],axis=1,inplace=True)

_test_dset = Dataset.from_pandas(df_test)
_test_dset_predicted = controller.predict_raw_dset(_test_dset,
                                                   do_filtering=True, # since we have some text filtering in the processing
                                                  )

-------------------- Start making predictions --------------------

df_test_predicted = _test_dset_predicted.to_pandas()

df_test_predicted.head()

	Title	Review Text	input_ids	attention_mask	pred_Division Name	pred_prob_Division Name	pred_Department Name	pred_prob_Department Name
0	perfect for work and play	perfect for work and play . this shirt works f...	[0, 20473, 13, 173, 8, 310, 479, 42, 6399, 136...	[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...	General	0.549556	Tops	0.999717
1		. i don't know why i had the opposite problem ...	[0, 4, 939, 218, 75, 216, 596, 939, 56, 5, 548...	[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...	General	0.668210	Bottoms	0.999585
2	great pants	great pants . thes e cords are great--lightwei...	[0, 12338, 9304, 479, 5, 29, 364, 37687, 32, 3...	[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...	General	0.629028	Bottoms	0.999531
3	surprisingly comfy for a button down	surprisingly comfy for a button down . i am a ...	[0, 33258, 3137, 24382, 13, 10, 6148, 159, 479...	[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...	General	0.530994	Tops	0.996693
4	short and small	short and small . the shirt is mostly a thick ...	[0, 20263, 8, 650, 479, 5, 6399, 16, 2260, 10,...	[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...	General	0.499403	Tops	0.997771

Let’s quickly check the f1 score to make sure everything works correctly

f1_score(true_labels,df_test_predicted['pred_Department Name'],average='macro')

0.7058640842784157

Predict top k results

_test_dset = Dataset.from_pandas(df_test)
_test_dset_predicted = controller.predict_raw_dset(_test_dset,
                                                   do_filtering=True,
                                                   topk=3
                                                  )

-------------------- Start making predictions --------------------

df_test_predicted = _test_dset_predicted.to_pandas()

df_test_predicted.head()

	Title	Review Text	input_ids	attention_mask	pred_Division Name	pred_prob_Division Name	pred_Department Name	pred_prob_Department Name
0	perfect for work and play	perfect for work and play . this shirt works f...	[0, 20473, 13, 173, 8, 310, 479, 42, 6399, 136...	[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...	[General, General Petite, Initmates]	[0.5495558, 0.38857713, 0.061867107]	[Tops, Intimate, Trend]	[0.9997173, 0.00027123763, 5.448324e-06]
1		. i don't know why i had the opposite problem ...	[0, 4, 939, 218, 75, 216, 596, 939, 56, 5, 548...	[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...	[General, General Petite, Initmates]	[0.6682097, 0.26135367, 0.0704367]	[Bottoms, Intimate, Trend]	[0.99958473, 0.00031657313, 6.8090965e-05]
2	great pants	great pants . thes e cords are great--lightwei...	[0, 12338, 9304, 479, 5, 29, 364, 37687, 32, 3...	[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...	[General, General Petite, Initmates]	[0.62902796, 0.29412356, 0.07684846]	[Bottoms, Intimate, Trend]	[0.9995307, 0.00037790896, 5.9614045e-05]
3	surprisingly comfy for a button down	surprisingly comfy for a button down . i am a ...	[0, 33258, 3137, 24382, 13, 10, 6148, 159, 479...	[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...	[General, General Petite, Initmates]	[0.530994, 0.384637, 0.084368974]	[Tops, Intimate, Dresses]	[0.99669266, 0.003254413, 2.1810652e-05]
4	short and small	short and small . the shirt is mostly a thick ...	[0, 20263, 8, 650, 479, 5, 6399, 16, 2260, 10,...	[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...	[General, General Petite, Initmates]	[0.49940288, 0.3735293, 0.12706791]	[Tops, Intimate, Trend]	[0.9977709, 0.0022051241, 1.0216948e-05]

# Since we have some metadatas (Title and Division Name), we need to define a dictionary containing those values
raw_content={'Review Text': 'This shirt is so comfortable I love it!',
             'Title': 'Great shirt'}

controller.data_store.num_proc=1

df_result = controller.predict_raw_text(raw_content,topk=3)

-------------------- Start making predictions --------------------

df_result

{'Review Text': ['great shirt . this shirt is so comfortable i love it !'],
 'Title': ['great shirt'],
 'input_ids': [[0,
   12338,
   6399,
   479,
   42,
   6399,
   16,
   98,
   3473,
   939,
   657,
   24,
   27785,
   2]],
 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]],
 'pred_Division Name': [['General', 'General Petite', 'Initmates']],
 'pred_prob_Division Name': [[0.564812958240509,
   0.3748474419116974,
   0.06033959984779358]],
 'pred_Department Name': [['Tops', 'Intimate', 'Trend']],
 'pred_prob_Department Name': [[0.9997146725654602,
   0.00027399769169278443,
   5.55193309992319e-06]]}