Sentiment Analysis: Pre-processing the Data

positive_validation   = positive[SPLIT:]
positive_training  = positive[:SPLIT]

negative_validation = negative[SPLIT:]
negative_training  = negative[:SPLIT]

train_x = positive_training + negative_training
validation_x = positive_validation + negative_validation

train_y = numpy.append(numpy.ones(len(positive_training)),
                       numpy.zeros(len(negative_training)))
validation_y  = numpy.append(numpy.ones(len(positive_validation)),
                             numpy.zeros(len(negative_validation)))

print(f"length of train_x {len(train_x):,}")
print(f"length of validation_x {len(validation_x):,}")

# UNQ_C2 (UNIQUE CELL IDENTIFIER, DO NOT EDIT)
# GRADED: Data generator
def data_generator(data_pos: list, data_neg: list, batch_size: int,
                   loop: bool, vocab_dict: dict, shuffle: bool=False):
    """Generates batches of data

    Args: 
       data_pos - Set of positive examples
       data_neg - Set of negative examples
       batch_size - number of samples per batch. Must be even
       loop - True or False
       vocab_dict - The words dictionary
       shuffle - Shuffle the data order

    Yield:
       inputs - Subset of positive and negative examples
       targets - The corresponding labels for the subset
       example_weights - An array specifying the importance of each example        
    """
    # make sure the batch size is an even number
    # to allow an equal number of positive and negative samples
    assert batch_size % 2 == 0

    # Number of positive examples in each batch is half of the batch size
    # same with number of negative examples in each batch
    n_to_take = batch_size // 2

    # Use pos_index to walk through the data_pos array
    # same with neg_index and data_neg
    pos_index = 0
    neg_index = 0

    len_data_pos = len(data_pos)
    len_data_neg = len(data_neg)

    # Get and array with the data indexes
    pos_index_lines = list(range(len_data_pos))
    neg_index_lines = list(range(len_data_neg))

    # shuffle lines if shuffle is set to True
    if shuffle:
        rnd.shuffle(pos_index_lines)
        rnd.shuffle(neg_index_lines)

    stop = False

    # Loop indefinitely
    while not stop:  

        # create a batch with positive and negative examples
        batch = []

        # First part: Pack n_to_take positive examples

        # Start from pos_index and increment i up to n_to_take
        for i in range(n_to_take):

            # If the positive index goes past the positive dataset length,
            if pos_index >= len_data_pos: 

                # If loop is set to False, break once we reach the end of the dataset
                if not loop:
                    stop = True;
                    break;

                # If user wants to keep re-using the data, reset the index
                pos_index = 0

                if shuffle:
                    # Shuffle the index of the positive sample
                    rnd.shuffle(pos_index_lines)

            # get the tweet as pos_index
            tweet = data_pos[pos_index_lines[pos_index]]

            # convert the tweet into tensors of integers representing the processed words
            tensor = tweet_to_tensor(tweet, vocab_dict)

            # append the tensor to the batch list
            batch.append(tensor)

            # Increment pos_index by one
            pos_index = pos_index + 1


        # Second part: Pack n_to_take negative examples

        # Using the same batch list, start from neg_index and increment i up to n_to_take
        for i in range(neg_index, n_to_take):

            # If the negative index goes past the negative dataset length,
            if neg_index > len_data_neg:

                # If loop is set to False, break once we reach the end of the dataset
                if not loop:
                    stop = True;
                    break;

                # If user wants to keep re-using the data, reset the index
                neg_index = 0

                if shuffle:
                    # Shuffle the index of the negative sample
                    rnd.shuffle(neg_index_lines)
            # get the tweet at neg_index
            tweet = data_neg[neg_index_lines[neg_index]]

            # convert the tweet into tensors of integers representing the processed words
            tensor = tweet_to_tensor(tweet, vocab_dict)

            # append the tensor to the batch list
            batch.append(tensor)

            # Increment neg_index by one
            neg_index += 1

        if stop:
            break;

        # Update the start index for positive data 
        # so that it's n_to_take positions after the current pos_index
        pos_index += n_to_take

        # Update the start index for negative data 
        # so that it's n_to_take positions after the current neg_index
        neg_index += n_to_take

        # Get the max tweet length (the length of the longest tweet) 
        # (you will pad all shorter tweets to have this length)
        max_len = max([len(t) for t in batch]) 


        # Initialize the input_l, which will 
        # store the padded versions of the tensors
        tensor_pad_l = []
        # Pad shorter tweets with zeros
        for tensor in batch:
            # Get the number of positions to pad for this tensor so that it will be max_len long
            n_pad = max_len - len(tensor)

            # Generate a list of zeros, with length n_pad
            pad_l = [0] * n_pad

            # concatenate the tensor and the list of padded zeros
            tensor_pad = tensor + pad_l

            # append the padded tensor to the list of padded tensors
            tensor_pad_l.append(tensor_pad)

        # convert the list of padded tensors to a numpy array
        # and store this as the model inputs
        inputs = numpy.array(tensor_pad_l)

        # Generate the list of targets for the positive examples (a list of ones)
        # The length is the number of positive examples in the batch
        target_pos = [1] * len(batch[:n_to_take])

        # Generate the list of targets for the negative examples (a list of zeros)
        # The length is the number of negative examples in the batch
        target_neg = [0] * len(batch[n_to_take:])

        # Concatenate the positve and negative targets
        target_l = target_pos + target_neg

        # Convert the target list into a numpy array
        targets = numpy.array(target_l)

        # Example weights: Treat all examples equally importantly.It should return an np.array. Hint: Use np.ones_like()
        example_weights = numpy.ones_like(targets)

        yield inputs, targets, example_weights

# Set the random number generator for the shuffle procedure
rnd = random
rnd.seed(30) 

# Create the training data generator
def train_generator(batch_size, shuffle = False):
    return data_generator(positive_training, negative_training,
                          batch_size, True, vocabulary, shuffle)

# Create the validation data generator
def val_generator(batch_size, shuffle = False):
    return data_generator(positive_validation, negative_validation,
                          batch_size, True, vocabulary, shuffle)

# Create the validation data generator
def test_generator(batch_size, shuffle = False):
    return data_generator(positive_validation, negative_validation, batch_size,
                          False, vocabulary, shuffle)

# Get a batch from the train_generator and inspect.
inputs, targets, example_weights = next(train_generator(4, shuffle=True))

# this will print a list of 4 tensors padded with zeros
print(f'Inputs: {inputs}')
print(f'Targets: {targets}')
print(f'Example Weights: {example_weights}')

tmp_data_gen = train_generator(batch_size = 4)

tmp_inputs, tmp_targets, tmp_example_weights = next(tmp_data_gen)

print(f"The inputs shape is {tmp_inputs.shape}")
print(f"The targets shape is {tmp_targets.shape}")
print(f"The example weights shape is {tmp_example_weights.shape}")

for i,t in enumerate(tmp_inputs):
    print(f"input tensor: {t}; target {tmp_targets[i]}; example weights {tmp_example_weights[i]}")

# python
from argparse import Namespace
from itertools import cycle

import random

# pypi
from nltk.corpus import twitter_samples

import attr
import numpy

# this project
from .processor import TwitterProcessor

Defaults = Namespace(
    split = 4000,
)

NLTK = Namespace(
    corpus="twitter_samples",
    negative = "negative_tweets.json",
    positive="positive_tweets.json",
)

SpecialTokens = Namespace(padding="__PAD__",
                          ending="__</e>__",
                          unknown="__UNK__")

SpecialIDs = Namespace(
    padding=0,
    ending=1,
    unknown=2,
)

@attr.s(auto_attribs=True)
class TensorBuilder:
    """converts tweets to tensors

    Args: 
     - split: where to split the training and validation data
    """
    split = Defaults.split
    _positive: list=None
    _negative: list=None
    _positive_training: list=None
    _negative_training: list=None
    _positive_validation: list=None
    _negative_validation: list=None
    _process: TwitterProcessor=None
    _vocabulary: dict=None
    _x_train: list=None

@attr.s(auto_attribs=True)
class TensorGenerator:
    """Generates batches of vectorized-tweets

    Args:
     converter: TensorBuilder object
     positive_data: list of positive data
     negative_data: list of negative data
     batch_size: the size for each generated batch     
     shuffle: whether to shuffle the generated data
     infinite: whether to generate data forever
    """
    converter: TensorBuilder
    positive_data: list
    negative_data: list
    batch_size: int
    shuffle: bool=True
    infinite: bool = True
    _positive_indices: list=None
    _negative_indices: list=None
    _positives: iter=None
    _negatives: iter=None