from argparse import Namespace
from functools import partial
from pathlib import Path

from eli5.sklearn import PermutationImportance

from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split, RandomizedSearchCV
from tabulate import tabulate

import eli5
import hvplot.pandas
import numpy
import pandas

from graeae import EmbedHoloviews, EnvironmentLoader, Timer

TIMER = Timer()

SLUG = "exercise-in-permutation-importance"
PATH = Path("../../files/posts/tutorials/")/SLUG
Plot = Namespace(
    width=1000,
    height=800,
    )
Embed = partial(EmbedHoloviews, folder_path=PATH)

ENVIRONMENT = EnvironmentLoader()

TABLE = partial(tabulate, tablefmt="orgtbl", headers="keys", showindex=False)

with TIMER:
    permutor = PermutationImportance(first_model, random_state=1).fit(
        x_validate, y_validate)

ipython_html = eli5.show_weights(permutor,
                                 feature_names=x_validate.columns.tolist())
table = pandas.read_html(ipython_html.data)[0]
print(TABLE(table))

data['absolute_change_longitude'] = abs(
    data.dropoff_longitude - data.pickup_longitude)
data['absolute_change_latitude'] = abs(
    data.dropoff_latitude - data.pickup_latitude)

features_2  = ['pickup_longitude',
               'pickup_latitude',
               'dropoff_longitude',
               'dropoff_latitude',
               'absolute_change_latitude',
               'absolute_change_longitude']

X = data[features_2]
new_x_train, new_x_validate, new_y_train, new_y_validate = train_test_split(
    X, y, random_state=1)

estimators = list(range(100, 250, 10))
max_depth = list(range(10, 50, 10)) + [None]
model = RandomForestRegressor()
search = RandomizedSearchCV(estimator=model,
                            param_distributions=grid,
                            n_jobs=-1,
                            random_state=1)
with TIMER:
    search.fit(new_x_train, new_y_train)

second_model = search.best_estimator_
print(f"Mean Cross-Validation Training R^2: {search.best_score_:0.2f}")
print(f"Training R^2: {second_model.score(new_x_train, new_y_train): 0.2f}")
print("Validation R^2: "
      f"{second_model.score(new_x_validate, new_y_validate):0.2f}")
print(search.best_params_)

permutor = PermutationImportance(second_model, random_state=1).fit(
    new_x_validate, new_y_validate)
ipython_html = eli5.show_weights(permutor,
                                 feature_names=new_x_validate.columns.tolist())
table = pandas.read_html(ipython_html.data)[0]
print(TABLE(table))

print(TABLE(new_x_train.sample(random_state=1).iloc[0].reset_index()))

print(new_x_validate.describe())

for column in ("pickup_longitude pickup_latitude dropoff_longitude "
               "dropoff_latitude absolute_change_latitude "
               "absolute_change_longitude").split():
    print(f"{column}: {new_x_validate[column].max() - new_x_validate[column].min():0.3f}")

data["bigger_pickup_longitude"] = data.pickup_longitude * 100
data["bigger_absolute_change_longitude"] = data.absolute_change_longitude * 100
features_3  = ['pickup_longitude',
               'pickup_latitude',
               'dropoff_longitude',
               'dropoff_latitude',
               'absolute_change_latitude',
               'absolute_change_longitude',
               'bigger_pickup_longitude',
               'bigger_absolute_change_longitude'
               ]

X = data[features_3]
big_x_train, big_x_validate, big_y_train, big_y_validate = train_test_split(
    X, y, random_state=1)
model = RandomForestRegressor()
search = RandomizedSearchCV(estimator=model,
                            param_distributions=grid,
                            n_jobs=-1,
                            random_state=1)
with TIMER:
    search.fit(big_x_train, big_y_train)
big_model = search.best_estimator_
print(f"Mean Cross-Validation Training R^2: {search.best_score_:0.2f}")
print(f"Training R^2: {big_model.score(big_x_train, big_y_train): 0.2f}")
print("Validation R^2: "
      f"{big_model.score(big_x_validate, big_y_validate):0.2f}")
print(search.best_params_)

permutor = PermutationImportance(big_model, random_state=1).fit(
    big_x_validate, big_y_validate)
ipython_html = eli5.show_weights(permutor,
                                 feature_names=big_x_validate.columns.tolist())
table = pandas.read_html(ipython_html.data)[0]
print(TABLE(table))

from functools import partial
from pathlib import Path

from graeae import CountPercentage, EmbedHoloviews, EnvironmentLoader

SLUG = "permutation-importance"
PATH = Path("files/posts/tutorials/")/SLUG
Embed = partial(EmbedHoloviews, folder_path=PATH)

ENVIRONMENT = EnvironmentLoader(path="posts/tutorials/.env")

TABLE = partial(tabulate, tablefmt="orgtbl", headers="keys", showindex=False)

path = Path(ENVIRONMENT["FIFA-2018"]).expanduser()
data = pandas.read_csv(path)

CountPercentage(data["Man of the Match"])()

data.loc[:, "Man of the Match"] = data["Man of the Match"] == "Yes"
CountPercentage(data["Man of the Match"])()

weights = table.Weight.str.split(expand=True)[0].astype(float)
table["weights"] = weights
plot = table.hvplot.bar(x="Feature", y="weights").opts(
    title="Permutation Importance (by Accuracy)",
    width=1000, height=800, xrotation=45)
output = Embed(plot=plot, file_name="permutation_importance")()

print(output)

from functools import partial
from typing import Any
from math import log2

from expects import (
    be_within,
    expect,
)
from tabulate import tabulate
import attr
import pandas

 def node_entropy(data: pandas.Series, debug: bool=False) -> float:
     """Calculate the entropy for a child node

     Args:
      data: target data filtered to match the child-node's class
      debug: emit values

     Returns:
      entropy for this node
     """
     if debug:
         print("calculating node-entropy")
     total = len(data)
     accumulated = 0
     for classification in data.unique():
         p = len(data[data == classification])/total
         if debug:
             print(f"\tclass: {classification}, p: {p} entropy: {p * log2(p)}")
         accumulated += p * log2(p)
     if debug:
         print(f"Node Entropy: {accumulated}")
     return -accumulated

def children_impurity(data: pandas.DataFrame, column: str, target: str,
                      impurity: object=node_entropy,
            debug: bool=False) -> float:
    """Calculate the entropy for the parent of child nodes

    Args:
     data: the container for the values to check
     column: the column to get the entropy
     target: the target column
     impurity: the function to calculate the impurity of the child
     debug: whether to print some intermediate values

    Returns:
     entropy for the data
    """
    if debug:
        print("Calculating entropy for child nodes")
    total = len(data)
    accumulator = 0
    for classification in data[column].unique():
        child = data[data[column] == classification][target]
        if debug:
            print(f"\tI_({classification}) = ({len(child)}/{total}) "
                  f"x {impurity(child)}")
        accumulator += (len(child)/total) * impurity(child)
    if debug:
        print(f"Child node entropy: {accumulator}")
    return accumulator

 def information_gain(data: pandas.Series, column: str, target: str,
                      debug: bool=False) -> float:
     """See how much entropy is removed using this node

     Args:
      data: source to check
      column: name of the column representing the parent node
      target: name of the column we are trying to predict
      debug: emit messages
     """
     return node_entropy(data[target], debug=debug) - children_impurity(
         data, column=column, target=target, debug=debug)

 @attr.s(auto_attribs=True, slots=True, frozen=True)
 class LoanColumns:
     owner: str = "Home Owner"
     married: str = "Marital Status"
     income: str = "Annual Income"
     defaulted: str = "Defaulted"

 LOANS = LoanColumns()

 loans = pandas.DataFrame({
     LOANS.owner: [True, False, False, True, False, False, True, False, False, False],
     LOANS.married: ["Single", "Married", "Single", "Married", "Divorced", "Single", "Divorced", "Single", "Married", "Single"],
     LOANS.income: [125000, 100000, 70000, 120000, 95000, 60000, 220000, 85000, 75000, 90000],
     LOANS.defaulted: [False, False, False, False, True, False, False, True, False, True],
 })
 print(TABLE(loans))

 impurity_parent = node_entropy(loans[LOANS.defaulted])
 print(f"I_parent: {impurity_parent:0.3f}")

 expect(impurity_parent).to(be_within(0.8812, 0.8813))

parent_gini = gini(loans[LOANS.defaulted])
print(f"I_parent = {parent_gini: 0.3f}")
expect(parent_gini).to(be_within(0.420, 0.421))

homeowner_gini = children_impurity(loans, LOANS.owner, LOANS.defaulted, gini, debug=True)
expect(homeowner_gini).to(be_within(0.342, 0.344))

def binary_gini(data: pandas.Series, column: str, target: str,
                classification: object, debug: bool=False) -> float:
    """Calculate the gini value for the data using one against many

    Args:
     data: source with qualitative values
     column: column with the classifications to test
     target: column with the classifications to predict
     classification: the classification to compare
     debug: whether to emit messages
    """
    total = len(data)
    classified = data[data[column] == classification]
    others = data[data[column] != classification]
    if debug:
        print(f"N({classification}/N) I({classification}) = {len(classified)/total * gini(classified[target]):0.3f}")
        print(f"N(!{classification}/N) I!({classification}) = {len(others)/total * gini(others[target]):0.3f}")
        print(f"I({classification}) = {((len(classified)/total) * gini(classified[target]) + (len(others)/total) * gini(others[target])):0.3f}")
    return ((len(classified)/total) * gini(classified[target])
            + (len(others)/total) * gini(others[target]))

@attr.s(auto_attribs=True, slots=True, frozen=True)
class MaritalStatus:
    single: str="Single"
    married: str="Married"
    divorced: str="Divorced"

status = MaritalStatus()

i_single = binary_gini(loans, LOANS.married, LOANS.defaulted, status.single,
                       debug=True)
print()
i_married = binary_gini(loans, LOANS.married, LOANS.defaulted, status.married,
                        debug=True)
print()
i_divorced = binary_gini(loans, LOANS.married, LOANS.defaulted,
                         status.divorced, debug=True)
expect(i_single).to(be_within(0.39, 0.41))
expect(i_divorced).to(be_within(0.39, 0.41))
expect(i_married).to(be_within(0.342, 0.344))

best = 0
best_split = None
for candidate, label in ((homeowner_gini, "Homeowner"),
                         (i_single, "Single"),
                         (i_married, "Married"),
                         (i_divorced, "Divorced")):
    delta = parent_gini - candidate
    if delta > best:
        best = delta
        best_split = label
    print(f"Delta {label} = {delta:0.3f}")
print(f"Best Split: {best_split}")

from argparse import Namespace
from functools import partial
from pathlib import Path
import re

from snorkel.analysis import get_label_buckets
from snorkel.labeling import labeling_function, LabelingFunction, LFAnalysis, PandasLFApplier
from snorkel.preprocess import preprocessor
from snorkel.labeling.lf.nlp import nlp_labeling_function
from sklearn.model_selection import train_test_split
from tabulate import tabulate
from textblob import TextBlob

import pandas

TABLE = partial(tabulate, tablefmt="orgtbl", headers="keys")

Comment = Namespace(
    is_ambiguous = -1,
    is_ham = 0,
    is_spam = 1
)

Data = Namespace(
    test_artist = "Shakira",
    development_size = 200,
    validation_size = 0.5,
    random_seed = 666,
)

Columns = Namespace(
    comment = "CONTENT",
    classification = "CLASS",
    artist = "artist",
)

@labeling_function(pre=[textblob_sentiment])
def textblob_polarity(row: pandas.Series) -> int:
    """decides if the comment is ham based on the polarity of the sentiment"""
    return Comment.is_ham if row.polarity > 0.9 else Comment.is_ambiguous

@labeling_function(pre=[textblob_sentiment])
def textblob_subjectivity(row: pandas.Series) -> int:
    """decides if the comment is ham based on the subjectivity"""
    return Comment.is_ham if row.subjectivity > 0.5 else Comment.is_ambiguous

labeling_functions = [textblob_polarity, textblob_subjectivity]
applier = PandasLFApplier(lfs=labeling_functions)
train_label_matrix = applier.apply(train, progress_bar=False)
development_label_matrix = applier.apply(development, progress_bar=False)

print(TABLE(LFAnalysis(train_label_matrix, labeling_functions).lf_summary()))

print(TABLE(LFAnalysis(development_label_matrix, labeling_functions).lf_summary(Y=development.CLASS.values)))

from argparse import Namespace
from functools import partial
from pathlib import Path
import random
import re

from nltk.corpus import wordnet
from sklearn import metrics
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from snorkel.augmentation import ApplyOnePolicy, PandasTFApplier, transformation_function
from snorkel.labeling import labeling_function, LabelModel, PandasLFApplier
from snorkel.slicing import slicing_function
from textblob import TextBlob
import hvplot.pandas
import nltk
import pandas

from graeae import CountPercentage, EmbedHoloviews

nltk.download("wordnet", quiet=True)

Embed = partial(EmbedHoloviews, folder_path="../../../files/posts/libraries/snorkel/snorkel-example-building-a-spam-dataset")

path = Path("~/data/datasets/texts/you_tube_comments/").expanduser()
sets = []
for name in path.glob("*.csv"):
    artist = name.stem.split()[-1]
    data = pandas.read_csv(name)
    data["artist"] = artist
    sets.append(data)
    print(artist)
data = pandas.concat(sets)

train, test = train_test_split(data)
print(train.shape)
print(test.shape)

train, development = train_test_split(train)
validation, test = train_test_split(test)
print(train.shape)
print(development.shape)
print(validation.shape)
print(test.shape)

grouped = train.groupby(["artist"]).agg({"COMMENT_ID": "count"}).reset_index().rename(columns={"COMMENT_ID": "Count"})
plot = grouped.hvplot.bar(x="artist", y="Count").opts(title="Comments by Artist", width=1000, height=800)
Embed(plot=plot, file_name="comments_by_artist")()

grouped = train.groupby(["artist", "CLASS"]).agg({"COMMENT_ID": "count"}).reset_index().rename(columns={"COMMENT_ID": "Count"})
plot = grouped.hvplot.bar(x="artist", y="Count", by="CLASS").opts(title="Comments by Artist and Class", width=1000, height=800)
Embed(plot=plot, file_name="comments_by_artist_and_class")()

Label = Namespace(
    abstain = -1,
    not_spam = 0,
    spam = 1,
)

@labeling_function()
def labeling_by_keyword(comment: pandas.Series) -> int:
    """Assume if the author refers to something he/she owns it's spam

    Args: 
     row with comment CONTENT

    Returns:
     label for the comment
    """
    return Label.spam if "my" in comment.CONTENT.lower() else Label.abstain

@labeling_function()
def label_check_out(comment) -> int:
    """check my/it/the out will be spam"""
    return Label.spam if re.search(r"check.*out", comment.CONTENT, flags=re.I) else Label.abstain

@labeling_function()
def label_short_comment(comment) -> int:
    """if a comment is short it's probably not spam"""
    return Label.not_spam if len(comment.CONTENT.split()) < 5 else Label.abstain

@labeling_function()
def label_positive_comment(comment) -> int:
    """If a comment is positive, we'll accept it"""
    return Label.not_spam if TextBlob(comment.CONTENT).sentiment.polarity > 0.3 else Label.abstain

labeling_functions = [labeling_by_keyword, label_check_out, label_short_comment, label_positive_comment]

applier = PandasLFApplier(labeling_functions)

label_matrix = applier.apply(train, progress_bar=False)

print(label_matrix.shape)
print(train.shape)

label_frame = pandas.DataFrame(label_matrix, columns=["keyword", "check_out", "short", "positive"])

re_framed = {}

for column in label_frame.columns:
    re_framed[column] = 

label_model = LabelModel(cardinality=2, verbose=True)
label_model.fit(label_matrix, n_epochs=500, log_freq=50, seed=0)
train["label"] = label_model.predict(L=label_matrix, tie_break_policy="abstain")

grouped = train.groupby(["label", "artist"]).agg({"COMMENT_ID": "count"}).reset_index().rename(columns={"COMMENT_ID": "count"})
plot = grouped.hvplot.bar(x="label", y="count", by="artist").opts(title="Label Counts", height=800, width=1000)
Embed(plot=plot, file_name="label_counts")()

train = train[train.label != Label.abstain]
CountPercentage(train.label)()

matched = sum(train.label == train.CLASS)
print(f"{matched/len(train): .2f}")

def synonyms_for(word: str) -> list:
    """get synonyms for word"""
    lemmas = set().union(*[synset.lemmas() for synset in wordnet.synsets(word)])
    return list(set(lemma.name().lower().replace("_" , " ") for lemma in lemmas) - {word})

@transformation_function()
def replace_word_with_synonym(comment: pandas.Series) -> pandas.Series:
    """Replace one of the words with a synonym

    Args:
     comment: row with a comment

    Returns:
     comment with a word replaced
    """
    tokens = comment.CONTENT.lower().split()
    index = random.choice(range(len(tokens)))
    synonyms = synonyms_for(tokens[index])
    if synonyms:
        comment.CONTENT = " ".join(tokens[:index] + [synonyms[0]] + tokens[index + 1 :])
    return comment

transform_policy = ApplyOnePolicy(n_per_original=2, keep_original=True)
transform_applier = PandasTFApplier([replace_word_with_synonym], transform_policy)
train_augmented = transform_applier.apply(train, progress_bar=False)

print(train_augmented[:3].CONTENT)

print(f"{len(train_augmented):,}")
train_augmented = train_augmented.drop_duplicates(subset="CONTENT")
print(f"{len(train_augmented):,}")
print(f"{len(train):,}")

vectorizer = CountVectorizer(ngram_range=(1, 2))
x_train = vectorizer.fit_transform(train.CONTENT)

classifier = LogisticRegression(solver="lbfgs")
classifier.fit(x_train, train.CLASS.values)

development_test = vectorizer.transform(development.CONTENT)
predicted = classifier.predict(development_test)

print(metrics.classification_report(development.CLASS, predicted, target_names=["not spam", "spam"]))

from sklearn.datasets import load_iris
import dabest
import pandas
import seaborn

iris = load_iris()

print(iris.DESCR)

data = pandas.DataFrame(iris.data, columns=iris.feature_names)
target = pandas.Series(iris.target)
names = dict(zip(range(len(iris.target_names)), iris.target_names))
data["species"] = target.map(names)

from sklearn.datasets import load_iris

from argparse import Namespace
from datetime import datetime
from functools import partial
from pathlib import Path

from sklearn.model_selection import train_test_split
import hvplot.pandas
import pandas

from graeae import CountPercentage, EmbedHoloviews

classified_as = Namespace(
    not_spam = 0,
    spam = 1,
    unknown = -1,
)

path = "../../files/posts/data/a-first-look-at-snorkel"
Embed = partial(EmbedHoloviews, folder_path=path)

path = Path("~/data/datasets/texts/you_tube_comments/").expanduser()
assert path.is_dir()

parts = []
for name in path.glob("*csv"):
    data = pandas.read_csv(name)    
    data["artist"] = name.name.split()[-1].split(".")[0]
    parts.append(data)

data = pandas.concat(parts)

Column = Namespace(
    comment_id = "comment_id",
    author = "author",
    datetime = "datetime",
    text = "text",
    label = "label",
    artist = "artist",
)
renames = {"COMMENT_ID": Column.comment_id,
           "AUTHOR": Column.author,
           "DATE": Column.datetime,
           "CONTENT": Column.text,
           "CLASS": Column.label}
data = data.rename(columns=renames)

counter = CountPercentage(data.label)
counter()

grouped = data.groupby([Column.artist]).agg({Column.label: "sum", Column.comment_id: "count"}).reset_index().rename(
    columns={Column.label: "spam", Column.comment_id: "total"})
grouped["ham"] = grouped.total - grouped.spam
plotter = grouped[[Column.artist, "spam", "ham"]]
plot = plotter.hvplot.bar(x=Column.artist, stacked=True, legend=True,).opts(
    title="Spam Counts",
    width=1000, height=800)
Embed(plot=plot, file_name="spam_counts")()

print(len(data[data[Column.datetime].isna()]))

with_date = data[~data[Column.datetime].isna()]
with_date.loc[:, Column.datetime] = pandas.to_datetime(with_date[Column.datetime])

with_date.loc[:, "Month"] = with_date[Column.datetime].apply(lambda date: datetime(date.year, date.month, 1))
group = with_date.groupby(["Month", Column.artist, Column.label]).agg(
    {Column.comment_id: "count"}).reset_index().rename(
        columns={Column.comment_id: "Count",
                 Column.artist: "Artist"})

spam = group[group[Column.label] == classified_as.spam]
ham = group[group[Column.label] == classified_as.not_spam]
spam_plot = spam.hvplot(x="Month", y="Count", by="Artist", label="Spam")
plot = spam_plot.opts(title="Monthly Spam By Artist", width=1000, height=800)
Embed(plot=plot, file_name="monthly_spam_by_artist")()