N-Gram: Building the Language Model

# python
from collections import defaultdict
from functools import partial

import random

# pypi
from tabulate import tabulate

import numpy
import pandas

TABLE = partial(tabulate, tablefmt="orgtbl", headers="keys")

n_gram_counts = {
    ('i', 'am', 'happy'): 2,
    ('am', 'happy', 'because'): 1}

# get count for an n-gram tuple
print(f"count of n-gram {('i', 'am', 'happy')}: {n_gram_counts[('i', 'am', 'happy')]}")

# check if n-gram is present in the dictionary
n_gram = ('i', 'am', 'learning')
status = "found" if n_gram in n_gram_counts else "missing"

print(f"n-gram {n_gram} {status}")

# update the count in the word count dictionary
n_gram_counts[n_gram] = 1
status = "found" if ('i', 'am', 'learning') in n_gram_counts else "missing"
print(f"n-gram {n_gram} {status}")

prefix = ('i', 'am', 'happy')
word = 'because'

# note here the syntax for creating a tuple for a single word
n_gram = prefix + (word,)
print(n_gram)

def single_pass_trigram_count_matrix(corpus: list) -> tuple:
    """
    Creates the trigram count matrix from the input corpus in a single pass through the corpus.

    Args:
       corpus: Pre-processed and tokenized corpus. 

    Returns:
       bigrams: list of all bigram prefixes, row index
       vocabulary: list of all found words, the column index
       count_matrix: pandas dataframe with bigram prefixes as rows, 
                     vocabulary words as columns 
                     and the counts of the bigram/word combinations (i.e. trigrams) as values
    """
    bigrams = []
    vocabulary = []
    count_matrix_dict = defaultdict(dict)

    # go through the corpus once with a sliding window
    for i in range(len(corpus) - 3 + 1):
        # the sliding window starts at position i and contains 3 words
        trigram = tuple(corpus[i : i + 3])

        bigram = trigram[0 : -1]
        if not bigram in bigrams:
            bigrams.append(bigram)        

        last_word = trigram[-1]
        if not last_word in vocabulary:
            vocabulary.append(last_word)

        if (bigram,last_word) not in count_matrix_dict:
            count_matrix_dict[bigram,last_word] = 0

        count_matrix_dict[bigram,last_word] += 1

    # convert the count_matrix to numpy.array to fill in the blanks
    count_matrix = numpy.zeros((len(bigrams), len(vocabulary)))
    for trigram_key, trigam_count in count_matrix_dict.items():
        count_matrix[bigrams.index(trigram_key[0]),
                     vocabulary.index(trigram_key[1])] = trigam_count

    # numpy.array to pandas dataframe conversion
    count_matrix = pandas.DataFrame(count_matrix, index=bigrams, columns=vocabulary)
    return bigrams, vocabulary, count_matrix

corpus = ['i', 'am', 'happy', 'because', 'i', 'am', 'learning', '.']

bigrams, vocabulary, count_matrix = single_pass_trigram_count_matrix(corpus)

print(TABLE(count_matrix))

vocabulary = ['i', 'am', 'happy', 'because', 'learning', '.', 'have', 'you', 'seen','it', '?']
starts_with = 'ha'

print(f'words in vocabulary starting with prefix: {starts_with}\n')
for word in (candidate for candidate in vocabulary
             if candidate.startswith(starts_with)):
    print(word)

def train_validation_test_split(data, train_percent, validation_percent):
    """
    Splits the input data to  train/validation/test according to the percentage provided

    Args:
       data: Pre-processed and tokenized corpus, i.e. list of sentences.
       train_percent: integer 0-100, defines the portion of input corpus allocated for training
       validation_percent: integer 0-100, defines the portion of input corpus allocated for validation

       Note: train_percent + validation_percent need to be <=100
             the reminder to 100 is allocated for the test set

    Returns:
       train_data: list of sentences, the training part of the corpus
       validation_data: list of sentences, the validation part of the corpus
       test_data: list of sentences, the test part of the corpus
    """
    # fixed seed here for reproducibility
    random.seed(87)

    # reshuffle all input sentences
    random.shuffle(data)

    train_size = int(len(data) * train_percent / 100)
    train_data = data[0:train_size]

    validation_size = int(len(data) * validation_percent / 100)
    validation_data = data[train_size:train_size + validation_size]

    test_data = data[train_size + validation_size:]

    return train_data, validation_data, test_data

p = 10 ** (-250)
M = 100

print(f"perplexity = {p ** (-1 / M):0.3f}")