from .models import *
from django import forms
from django.conf import settings
from os.path import join as join_path

from django.contrib.auth import get_user_model
from django.core.exceptions import ValidationError
from django.utils.translation import gettext_lazy as _

class UserHealthInfoForm(forms.ModelForm):

    Gender = forms.CharField(
        # label='Your gender',
        widget=forms.TextInput(
            attrs={

                'type': 'text',
                'id': 'gender',
                'name': 'gender',
                'class': 'form-control bg-white border-md pl-3',
                'placeholder': 'Your gender',
            }
        )
    )

    age = forms.CharField(
        # label='Your age',
        widget=forms.EmailInput(
            attrs={

                'type': 'text',
                'id': 'age',
                'name': 'age',
                'class': 'form-control bg-white border-md pl-3',
                'placeholder': 'Your age',
            }
        )
    )

    blood_group = forms.CharField(
        # label='Your blood group',
        widget=forms.TextInput(
            attrs={

                'id': 'blood-group',
                'name': 'blood-group',
                'class': 'form-control bg-white border-md pl-3',
                'placeholder': 'Your blood group',
            }
        )
    )

    genotype = forms.CharField(
        # label='Your genotype',
        widget=forms.TextInput(
            attrs={

                'id': 'genotype',
                'name': 'genotype',
                'class': 'form-control bg-white border-md pl-3',
                'placeholder': 'Your genotype',
            }
        )
    )

    ebola_status = forms.BooleanField(
        required=False,
        label='Do you have ebola currently?',
        widget=forms.CheckboxInput(
            attrs={

                'class': 'm-2',
                'id': 'ebolaStatus',
                # 'checked': 'checked',
            }
        )
    )

    malaria_status = forms.BooleanField(
        required=False,
        label='Do you have malaria currently?',
        widget=forms.CheckboxInput(
            attrs={

                'class': 'm-2',
                'id': 'malariaStatus',
                # 'checked': 'checked',
            }
        )
    )

    previous = forms.BooleanField(
        required=False,
        label='Do you have any previous health challenges?',
        widget=forms.CheckboxInput(
            attrs={

                'class': 'm-2',
                'id': 'previousHealthIssues',
                # 'checked': 'checked',
                'onclick': 'yesnoCheck()',
            }
        )
    )

    previous_illness1 = forms.CharField(
        required=False,
        label='What was your illness?',
        widget=forms.TextInput(
            attrs={

                'maxlength': '20',
                'style': 'display:none;',
                'id': 'previous-illness1',
                'name': 'previous-illness1',
                'class': 'form-control bg-white border-md pl-3',
                'placeholder': 'What was your illness?',
            }
        )
    )

    previous_illness2 = forms.CharField(
        required=False,
        label='Subject',
        widget=forms.TextInput(
            attrs={

                'style': 'display:none;',
                'id': 'previous-illness2',
                'name': 'previous-illness2',
                'class': 'form-control bg-white border-md pl-3',
                'placeholder': 'What was your illness?',
            }
        )
    )

    previous_illness3 = forms.CharField(
        required=False,
        label='Write your message here',
        widget=forms.TextInput(
            attrs={

                'style': 'display:none;',
                'id': 'previous-illness3',
                'name': 'previous-illness3',
                'class': 'form-control bg-white border-md pl-3',
                'placeholder': 'What was your illness?',
            }
        )
    )

    previous_illness4 = forms.CharField(
        required=False,
        label='Write your message here',
        widget=forms.TextInput(
            attrs={

                'style': 'display:none;',
                'id': 'previous-illness4',
                'name': 'previous-illness4',
                'class': 'form-control bg-white border-md pl-3',
                'placeholder': 'What was your illness?',
            }
        )
    )

    previous_illness5 = forms.CharField(
        required=False,
        label='Write your message here',
        widget=forms.TextInput(
            attrs={

                'style': 'display:none;',
                'id': 'previous-illness5',
                'name': 'previous-illness5',
                'class': 'form-control bg-white border-md pl-3',
                'placeholder': 'What was your illness?',
            }
        )
    )

    class Meta:
        model = UserHealthInfo
        fields = (
            'Gender',
            'age',
            'blood_group',
            'genotype',
            'ebola_status',
            'malaria_status',
            'previous',
            'previous_illness1',
            'previous_illness2',
            'previous_illness3',
            'previous_illness4',
            'previous_illness5',
        )

    def save(self, request, commit=True):
        UserHealthInfo_instance = super(UserHealthInfoForm, self).save(commit=False)
        UserHealthInfo_instance.user = request.user
        UserHealthInfo_instance.Gender = self.cleaned_data['Gender']
        UserHealthInfo_instance.age = self.cleaned_data['age']
        UserHealthInfo_instance.blood_group = self.cleaned_data['blood_group']
        UserHealthInfo_instance.genotype = self.cleaned_data['genotype']
        UserHealthInfo_instance.ebola_status = self.cleaned_data['ebola_status']
        UserHealthInfo_instance.malaria_status = self.cleaned_data['malaria_status']
        UserHealthInfo_instance.previous = self.cleaned_data['previous']
        UserHealthInfo_instance.previous_illness1 = self.cleaned_data['previous_illness1']
        UserHealthInfo_instance.previous_illness2 = self.cleaned_data['previous_illness2']
        UserHealthInfo_instance.previous_illness3 = self.cleaned_data['previous_illness3']
        UserHealthInfo_instance.previous_illness4 = self.cleaned_data['previous_illness4']
        UserHealthInfo_instance.previous_illness5 = self.cleaned_data['previous_illness5']

        # try:
        #     UserHealthInfo_instance.visitorphonenumber = self.cleaned_data['visitorphonenumber']
        # except Exception as Error:
        #     raise (Error)

        if commit:
            try:
                UserHealthInfo_instance.save()
            except Exception as Error:
                raise (Error)

        return UserHealthInfo_instance


import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import os
import gc
import matplotlib.pyplot as plt
import datetime
# import seaborn as sns
# %matplotlib inline

# pal = sns.color_palette()

# print('# File sizes')
# for f in os.listdir('../input'):
#     if 'zip' not in f:
#         print(f.ljust(30) + str(round(os.path.getsize('../input/' + f) / 1000000, 2)) + 'MB')

df_train = pd.read_csv('kaggle/Quora_Question_Pairs/Data/train.csv')
# df_train.head()

# print('Total number of question pairs for training: {}'.format(len(df_train)))
# print('Duplicate pairs: {}%'.format(round(df_train['is_duplicate'].mean()*100, 2)))
# qids = pd.Series(df_train['qid1'].tolist() + df_train['qid2'].tolist())
# print('Total number of questions in the training data: {}'.format(len(
#     np.unique(qids))))
# print('Number of questions that appear multiple times: {}'.format(np.sum(qids.value_counts() > 1)))
#
# plt.figure(figsize=(12, 5))
# plt.hist(qids.value_counts(), bins=50)
# plt.yscale('log', nonposy='clip')
# plt.title('Log-Histogram of question appearance counts')
# plt.xlabel('Number of occurences of question')
# plt.ylabel('Number of questions')

# from sklearn.metrics import log_loss
#
# p = df_train['is_duplicate'].mean() # Our predicted probability
# print('Predicted score:', log_loss(df_train['is_duplicate'], np.zeros_like(df_train['is_duplicate']) + p))
#
df_test = pd.read_csv('kaggle/Quora_Question_Pairs/Data/test.csv')
# sub = pd.DataFrame({'test_id': df_test['test_id'], 'is_duplicate': p})
# sub.to_csv('naive_submission.csv', index=False)
# sub.head()
#
# df_test = pd.read_csv('../input/test.csv')
# df_test.head()
#
# print('Total number of question pairs for testing: {}'.format(len(df_test)))

train_qs = pd.Series(df_train['question1'].tolist() + df_train['question2'].tolist()).astype(str)
# test_qs = pd.Series(df_test['question1'].tolist() + df_test['question2'].tolist()).astype(str)

# dist_train = train_qs.apply(len)
# dist_test = test_qs.apply(len)
# plt.figure(figsize=(15, 10))
# plt.hist(dist_train, bins=200, range=[0, 200], color=pal[2], normed=True, label='train')
# plt.hist(dist_test, bins=200, range=[0, 200], color=pal[1], normed=True, alpha=0.5, label='test')
# plt.title('Normalised histogram of character count in questions', fontsize=15)
# plt.legend()
# plt.xlabel('Number of characters', fontsize=15)
# plt.ylabel('Probability', fontsize=15)
#
# print('mean-train {:.2f} std-train {:.2f} mean-test {:.2f} std-test {:.2f} max-train {:.2f} max-test {:.2f}'.format(dist_train.mean(),
#                           dist_train.std(), dist_test.mean(), dist_test.std(), dist_train.max(), dist_test.max()))
#
# dist_train = train_qs.apply(lambda x: len(x.split(' ')))
# dist_test = test_qs.apply(lambda x: len(x.split(' ')))

# plt.figure(figsize=(15, 10))
# plt.hist(dist_train, bins=50, range=[0, 50], color=pal[2], normed=True, label='train')
# plt.hist(dist_test, bins=50, range=[0, 50], color=pal[1], normed=True, alpha=0.5, label='test')
# plt.title('Normalised histogram of word count in questions', fontsize=15)
# plt.legend()
# plt.xlabel('Number of words', fontsize=15)
# plt.ylabel('Probability', fontsize=15)
#
# print('mean-train {:.2f} std-train {:.2f} mean-test {:.2f} std-test {:.2f} max-train {:.2f} max-test {:.2f}'.format(dist_train.mean(),
#                           dist_train.std(), dist_test.mean(), dist_test.std(), dist_train.max(), dist_test.max()))
#
# from wordcloud import WordCloud
# cloud = WordCloud(width=1440, height=1080).generate(" ".join(train_qs.astype(str)))
# plt.figure(figsize=(20, 15))
# plt.imshow(cloud)
# plt.axis('off')

# qmarks = np.mean(train_qs.apply(lambda x: '?' in x))
# math = np.mean(train_qs.apply(lambda x: '[math]' in x))
# fullstop = np.mean(train_qs.apply(lambda x: '.' in x))
# capital_first = np.mean(train_qs.apply(lambda x: x[0].isupper()))
# capitals = np.mean(train_qs.apply(lambda x: max([y.isupper() for y in x])))
# numbers = np.mean(train_qs.apply(lambda x: max([y.isdigit() for y in x])))
#
# print('Questions with question marks: {:.2f}%'.format(qmarks * 100))
# print('Questions with [math] tags: {:.2f}%'.format(math * 100))
# print('Questions with full stops: {:.2f}%'.format(fullstop * 100))
# print('Questions with capitalised first letters: {:.2f}%'.format(capital_first * 100))
# print('Questions with capital letters: {:.2f}%'.format(capitals * 100))
# print('Questions with numbers: {:.2f}%'.format(numbers * 100))

from nltk.corpus import stopwords

stops = set(stopwords.words("english"))

replace_dict = {
    r"what's": "what is ",
    r"\'s": " ",
    r"\'ve": " have ",
    r"can't": "can not ",
    r"n't": " not ",
    r"i'm": "i am",
    r" m ": " am ",
    r"\'re": " are ",
    r"\'d": " would ",
    r"\'ll": " will ",
    r"60k": " 60000 ",
    r" e g ": " eg ",
    r" b g ": " bg ",
    r"\0s": "0",
    r" 9 11 ": "911",
    r"e-mail": "email",
    r"\s{2,}": " ",
    r"quikly": "quickly",
    r" usa ": " america ",
    r" u s ": " america ",
    r" uk ": " england ",
    r" us ": "america",
    # r"india": "India",
    # r"switzerland": "Switzerland",
    # r"china": "China",
    # r"chinese": "Chinese",
    r"imrovement": "improvement",
    r"intially": "initially",
    # r"quora": "Quora",
    r" dms ": "direct messages ",
    r"demonitization": "demonetization",
    r"actived": "active",
    r"kms": " kilometers ",
    r" cs ": " computer science ",
    r" upvotes ": " up votes ",
    r" iphone ": " phone ",
    r"\0rs ": " rs ",
    r"calender": "calendar",
    # r"ios": "operating system",
    # r"gps": "GPS",
    # r"gst": "GST",
    r"programing": "programming",
    r"bestfriend": "best friend",
    # r"dna": "DNA",
    r"III": "3",
    # r"Astrology": "astrology",
    # r"Method": "method",
    # r"Find": "find",
    # r"banglore": "Banglore",
    r" j k ": " jk ",
    r"[^A-Za-z0-9^,!.\/'+-=]": " "  # 删除特殊字符
}
import re
def clean_text(line):
    names = ["question1", "question2"]
    for name in names:
        l = line[name]
        l = str(l).lower()

        ## replace other words
        for k,v in replace_dict.items():
            l = re.sub(k, v, l)

        line[name] = l
    return line

import nltk
from nltk.tokenize import TreebankWordTokenizer, WordPunctTokenizer, WhitespaceTokenizer
from nltk import word_tokenize
from nltk.stem import PorterStemmer
from nltk.stem.wordnet import WordNetLemmatizer
from nltk import pos_tag

stemmer_type = "snowball"
if stemmer_type == "porter":
    english_stemmer = nltk.stem.PorterStemmer()
elif stemmer_type == "snowball":
    english_stemmer = nltk.stem.SnowballStemmer('english')
def stem_tokens(tokens, stemmer):
    stemmed = []
    for token in tokens:
        try:
            stemmed.append(stemmer.stem(token))
        except (e) as e:    # python 3.6 可能会报错，3.5.2 没问题
            print('exception is: ', e)
    return stemmed
# 1.     CC      Coordinating conjunction
# 2.     CD     Cardinal number
# 3.     DT     Determiner
# 4.     EX     Existential there
# 5.     FW     Foreign word
# 6.     IN     Preposition or subordinating conjunction
# 7.     JJ     Adjective
# 8.     JJR     Adjective, comparative
# 9.     JJS     Adjective, superlative
# 10.     LS     List item marker
# 11.     MD     Modal
# 12.     NN     Noun, singular or mass
# 13.     NNS     Noun, plural
# 14.     NNP     Proper noun, singular
# 15.     NNPS     Proper noun, plural
# 16.     PDT     Predeterminer
# 17.     POS     Possessive ending
# 18.     PRP     Personal pronoun
# 19.     PRP$     Possessive pronoun
# 20.     RB     Adverb
# 21.     RBR     Adverb, comparative
# 22.     RBS     Adverb, superlative
# 23.     RP     Particle
# 24.     SYM     Symbol
# 25.     TO     to
# 26.     UH     Interjection
# 27.     VB     Verb, base form
# 28.     VBD     Verb, past tense
# 29.     VBG     Verb, gerund or present participle
# 30.     VBN     Verb, past participle
# 31.     VBP     Verb, non-3rd person singular present
# 32.     VBZ     Verb, 3rd person singular present
# 33.     WDT     Wh-determiner
# 34.     WP     Wh-pronoun
# 35.     WP$     Possessive wh-pronoun
# 36.     WRB     Wh-adverb
wordnet_tags = ['n', 'v']
lemmatizer = WordNetLemmatizer()
def lemmatize(token, tag):
    if tag[0].lower() in wordnet_tags:
        return lemmatizer.lemmatize(token, tag[0].lower())
    return token

token_pattern = r"(?u)\b\w\w+\b"
# token_pattern = r'\w{1,}'
# token_pattern = r"\w+"
# token_pattern = r"[\w']+"
def preprocess_data(line, token_pattern=token_pattern, encode_digit=False):
    line = str(line).lower()
    # token_pattern = re.compile(token_pattern, flags = re.UNICODE | re.LOCALE)
    ## tokenize
    # tokens = [x.lower() for x in token_pattern.findall(line)]
    # tokens = line.split()
    # tokens = WhitespaceTokenizer().tokenize(line)
    # # stem
    # tokens_stemmed = stem_tokens(tokens, english_stemmer)
    # if True:
    #     tokens_stemmed = [x for x in tokens_stemmed if x not in stopwords]

    tagged_corpus = pos_tag(word_tokenize(line))
    tokens_lemmatize = [lemmatize(token, tag) for token, tag in tagged_corpus]
    return tokens_lemmatize
global ind
ind = 0
def word_match_share(row):
    row = clean_text(row)
    global ind
    q1words = {}
    q2words = {}
    if ind < 5:
        ind += 1
        print(preprocess_data(str(row['question1'])))
    for word in preprocess_data(str(row['question1'])):# str(row['question1']).lower().split()
        if word not in stops:
            q1words[word] = 1
    for word in preprocess_data(str(row['question2'])):
        if word not in stops:
            q2words[word] = 1
    if len(q1words) == 0 or len(q2words) == 0:
        # The computer-generated chaff includes a few questions that are nothing but stopwords
        return 0
    shared_words_in_q1 = [w for w in q1words.keys() if w in q2words]
    shared_words_in_q2 = [w for w in q2words.keys() if w in q1words]
    R = (len(shared_words_in_q1) + len(shared_words_in_q2))/(len(q1words) + len(q2words))
    return R

# plt.figure(figsize=(15, 5))
train_word_match = df_train.apply(word_match_share, axis=1, raw=True)
# plt.hist(train_word_match[df_train['is_duplicate'] == 0], bins=20, normed=True, label='Not Duplicate')
# plt.hist(train_word_match[df_train['is_duplicate'] == 1], bins=20, normed=True, alpha=0.7, label='Duplicate')
# plt.legend()
# plt.title('Label distribution over word_match_share', fontsize=15)
# plt.xlabel('word_match_share', fontsize=15)

from collections import Counter

# If a word appears only once, we ignore it completely (likely a typo)
# Epsilon defines a smoothing constant, which makes the effect of extremely rare words smaller
def get_weight(count, eps=10000, min_count=3):
    if count < min_count:
        return 0
    else:
        return 1 / (count + eps)

eps = 5000
words = (" ".join(train_qs)).lower().split()
counts = Counter(words)
weights = {word: get_weight(count) for word, count in counts.items()}

# print('Most common words and weights: \n')
# print(sorted(weights.items(), key=lambda x: x[1] if x[1] > 0 else 9999)[:10])
# print('\nLeast common words and weights: ')
# (sorted(weights.items(), key=lambda x: x[1], reverse=True)[:10])

def tfidf_word_match_share(row):
    row = clean_text(row)
    q1words = {}
    q2words = {}
    for word in preprocess_data(str(row['question1'])):
        if word not in stops:
            q1words[word] = 1
    for word in preprocess_data(str(row['question2'])):
        if word not in stops:
            q2words[word] = 1
    if len(q1words) == 0 or len(q2words) == 0:
        # The computer-generated chaff includes a few questions that are nothing but stopwords
        return 0

    shared_weights = [weights.get(w, 0) for w in q1words.keys() if w in q2words] + [weights.get(w, 0) for w in
                                                                                q2words.keys() if w in q1words]
    total_weights = [weights.get(w, 0) for w in q1words] + [weights.get(w, 0) for w in q2words]

    R = np.sum(shared_weights) / np.sum(total_weights)
    return R

# plt.figure(figsize=(15, 5))
tfidf_train_word_match = df_train.apply(tfidf_word_match_share, axis=1, raw=True)
# plt.hist(tfidf_train_word_match[df_train['is_duplicate'] == 0].fillna(0), bins=20, normed=True, label='Not Duplicate')
# plt.hist(tfidf_train_word_match[df_train['is_duplicate'] == 1].fillna(0), bins=20, normed=True, alpha=0.7, label='Duplicate')
# plt.legend()
# plt.title('Label distribution over tfidf_word_match_share', fontsize=15)
# plt.xlabel('word_match_share', fontsize=15)

# from sklearn.metrics import roc_auc_score
# print('Original AUC:', roc_auc_score(df_train['is_duplicate'], train_word_match))
# print('   TFIDF AUC:', roc_auc_score(df_train['is_duplicate'], tfidf_train_word_match.fillna(0)))

# from sklearn.metrics.pairwise import cosine_similarity as cs
# from sklearn.metrics.pairwise import manhattan_distances as md
# from sklearn.metrics.pairwise import euclidean_distances as ed
# from sklearn.metrics import jaccard_similarity_score as jsc
# from sklearn.neighbors import DistanceMetric
# from sklearn.preprocessing import MinMaxScaler
#
# minkowski_dis = DistanceMetric.get_metric('minkowski')
# mms_scale_man = MinMaxScaler()
# mms_scale_euc = MinMaxScaler()
# mms_scale_mink = MinMaxScaler()
#
# def get_similarity_values(q1_csc, q2_csc):
#     cosine_sim = []
#     manhattan_dis = []
#     eucledian_dis = []
#     jaccard_dis = []
#     minkowsk_dis = []
#
#     for i, j in zip(q1_csc, q2_csc):
#         sim = cs(i, j)
#         cosine_sim.append(sim[0][0])
#         sim = md(i, j)
#         manhattan_dis.append(sim[0][0])
#         sim = ed(i, j)
#         eucledian_dis.append(sim[0][0])
#         i_ = i.toarray()
#         j_ = j.toarray()
#         try:
#             sim = jsc(i_, j_)
#             jaccard_dis.append(sim)
#         except:
#             jaccard_dis.append(0)
#
#         sim = minkowski_dis.pairwise(i_, j_)
#         minkowsk_dis.append(sim[0][0])
#
#     return cosine_sim, manhattan_dis, eucledian_dis, jaccard_dis, minkowsk_dis
#
#
# cosine_sim, manhattan_dis, eucledian_dis, jaccard_dis, minkowsk_dis = get_similarity_values(q1_csc, q2_csc)
# print("cosine_sim sample= \n", cosine_sim[0:2])
# print("manhattan_dis sample = \n", manhattan_dis[0:2])
# print("eucledian_dis sample = \n", eucledian_dis[0:2])
# print("jaccard_dis sample = \n", jaccard_dis[0:2])
# print("minkowsk_dis sample = \n", minkowsk_dis[0:2])
#
# eucledian_dis_array = np.array(eucledian_dis).reshape(-1, 1)
# manhattan_dis_array = np.array(manhattan_dis).reshape(-1, 1)
# minkowsk_dis_array = np.array(minkowsk_dis).reshape(-1, 1)
#
# manhattan_dis_array = mms_scale_man.fit_transform(manhattan_dis_array)
# eucledian_dis_array = mms_scale_euc.fit_transform(eucledian_dis_array)
# minkowsk_dis_array = mms_scale_mink.fit_transform(minkowsk_dis_array)
#
# eucledian_dis = eucledian_dis_array.flatten()
# manhattan_dis = manhattan_dis_array.flatten()
# minkowsk_dis = minkowsk_dis_array.flatten()

# First we create our training and testing data
x_train = pd.DataFrame()
x_test = pd.DataFrame()
x_train['word_match'] = train_word_match
x_train['tfidf_word_match'] = tfidf_train_word_match
x_test['word_match'] = df_test.apply(word_match_share, axis=1, raw=True)
x_test['tfidf_word_match'] = df_test.apply(tfidf_word_match_share, axis=1, raw=True)

# train_comb = train_comb.drop(['id', 'is_duplicate'], axis = 1)
# test_comb = test_comb.drop(['id'], axis=1)
# x_train = pd.concat([x_train, train_comb], axis=1)
# x_test = pd.concat([x_test, test_comb], axis=1)

y_train = df_train['is_duplicate'].values
# pos_new/(pos_new + neg) = 0.165 ->  need reduce (pos - pos_new)
import numpy as np
need_reduce = y_train[y_train == 1].shape[0] - np.round(0.165 * y_train[y_train == 0].shape[0] / (1 - 0.165))
pos_ids = df_train['id'][y_train == 1]
random_seed = datetime.datetime.now().year + 1000 * (2 + 1)
pos_ids = pos_ids.sample(int(need_reduce), random_state=random_seed)
flags = df_train['id'].apply(lambda x: False if x in pos_ids else True)
y_train = df_train['is_duplicate'][flags].values
x_train = x_train[flags]

# pos_train = x_train[y_train == 1]
# neg_train = x_train[y_train == 0]

# Now we oversample the negative class
# There is likely a much more elegant way to do this...
# p = 0.165
# scale = ((len(pos_train) / (len(pos_train) + len(neg_train))) / p) - 1
# while scale > 1:
#     neg_train = pd.concat([neg_train, neg_train])
#     scale -=1
# neg_train = pd.concat([neg_train, neg_train[:int(scale * len(neg_train))]])
# print(len(pos_train) / (len(pos_train) + len(neg_train)))
#
# x_train = pd.concat([pos_train, neg_train])
# y_train = (np.zeros(len(pos_train)) + 1).tolist() + np.zeros(len(neg_train)).tolist()
# del pos_train, neg_train

# Finally, we split some of the data off for validation
from sklearn.cross_validation import train_test_split
# weight = np.abs(y_train - 1.2 + y_train.mean())
x_train2, x_valid, y_train2, y_valid = train_test_split(x_train, y_train, test_size=0.2, random_state=4242)

import xgboost as xgb

# Set our parameters for xgboost
params = {}
params['objective'] = 'binary:logistic'
params['eval_metric'] = 'logloss'
params['eta'] = 0.02
params['max_depth'] = 4

d_train = xgb.DMatrix(x_train2, label=y_train2)
d_valid = xgb.DMatrix(x_valid, label=y_valid)   # , weight=w_valid
d_test = xgb.DMatrix(x_test)
watchlist = [(d_train, 'train'), (d_valid, 'valid')]
bst = xgb.train(params, d_train, 600, watchlist, early_stopping_rounds=50, verbose_eval=10)

preds = bst.predict(d_test)
result = pd.DataFrame({"test_id": df_test['test_id'], "is_duplicate": preds})
result.to_csv("kaggle/Quora_Question_Pairs/temp/xgb_train_result.csv", index=False)
result.head()

from sklearn.grid_search import GridSearchCV
from sklearn.linear_model import Lasso
from sklearn.metrics import log_loss
from sklearn.preprocessing import  MinMaxScaler
# param_grid_lasso = {
#     'alpha': [0.0005, 0.001, 0.005, 0.006]
# }
# lasso = GridSearchCV(
#     estimator=Lasso(
#         alpha=0.0005,
#         fit_intercept=True,
#         normalize=False,
#         precompute=False,
#         copy_X=True,
#         max_iter=2000,
#         tol=1e-4,
#         warm_start=False,
#         positive=False,
#         random_state=None,
#         selection='cyclic'),
#     param_grid=param_grid_lasso,
#     # scoring='neg_log_loss',
#     n_jobs=4,
#     iid=False,
#     cv=5)
# lasso.fit(x_train, y_train)
# # lasso.predict_proba()
# y_preds = lasso.predict(x_train)
# lasso.grid_scores_, lasso.best_params_, lasso.best_score_
# min_max_scaler = MinMaxScaler()
# y_preds = min_max_scaler.fit_transform(y_preds)
# score = log_loss(y_train, y_preds)

from sklearn.linear_model import LogisticRegression
param_grid = {
    "C": [0.01, 0.05, 0.1, 0.5, 1.0]}
logistic = GridSearchCV(
    estimator=LogisticRegression(
        penalty='l2', dual=False,
        tol=1e-4,
        C=1.0,
        fit_intercept=True,
        intercept_scaling=1,
        class_weight=None,
        random_state=None,
        solver='liblinear',
        max_iter=100,
        multi_class='ovr',
        verbose=0,
        warm_start=False,
        n_jobs=1
    ),
    cv=5,
    param_grid=param_grid)
logistic.fit(x_train, y_train)
logistic.grid_scores_, logistic.best_params_, logistic.best_score_


import webbrowser
import os
import re
from xml.etree import ElementTree
import tkinter as tk
from tkinter import *
from tkinter import messagebox
from tkinter import filedialog
from tkinter import ttk

#################################################
# LISTA SIMPLE LINEAS DE PRODUCCION
#################################################
class linked_list_lineas_produccion:
    def __init__(self):
        self.head = None

    def insertar(self, id, CantidadComponentes, TiempoEnsamblaje, Componentes):
        if not self.head:
            self.head = nodeLSLP(id=id, CantidadComponentes=CantidadComponentes, TiempoEnsamblaje=TiempoEnsamblaje, Componentes=Componentes)
            return
        current = self.head
        while current.next:
            current = current.next
        current.next = nodeLSLP(id=id, CantidadComponentes=CantidadComponentes, TiempoEnsamblaje=TiempoEnsamblaje, Componentes=Componentes)

    def mostrar(self):
        actual = self.head
        while actual != None:
            print(actual.id, end="=>")
            actual = actual.next

    def buscar(self, id):
        actual = self.head

        while actual != None:
            if actual.id == id:
                return actual
            actual = actual.next
        return False

    def eliminar(self, id):
        current = self.head
        previous = None

        while current and current.id != id:
            previous = current
            current = current.next

            if previous is None:
                self.head = current.next
            elif current:
                previous.next = current.next
                current.next = None

    def imprimir(self):
        if self.head is None:
            return
        Actual = self.head
        print(Actual.id, end=") ")
        print(Actual.CantidadComponentes)
        while Actual.next != None:
            Actual = Actual.next
            print(Actual.id, end=") ")
            print(Actual.CantidadComponentes)

# Nodo Lista Simple Lineas Produccion
class nodeLSLP:
    def __init__(self, id=None, CantidadComponentes=None, TiempoEnsamblaje = None, Componentes=None ,next=None):
        self.id = id
        self.CantidadComponentes = CantidadComponentes
        self.TiempoEnsamblaje = TiempoEnsamblaje
        self.Componentes = Componentes
        self.next = next

#################################################
# LISTA SIMPLE COMPONENTES DE LINEAS DE PRODUCCION
#################################################
class linked_list_de_componentes_lineas_produccion:
  def __init__ (self, head=None):
    self.head = head
    self.last = head
    self.size = 0

  def insertarCabeza (self, id):
    if self.size == 0:
      self.head = nodeLSCLP(id=id)
      self.last = self.head
    else:
      new_node = nodeLSCLP(id=id, next=self.head)
      self.head.previous = new_node
      self.head = new_node
    self.size += 1

  def insertarFinal(self, id):
      if self.size == 0:
          self.head = nodeLSCLP(id=id)
      else:
          new_node = nodeLSCLP(id=id)
          temp = self.head
          while (temp.next != None):
              temp = temp.next

          temp.next = new_node
          new_node.prev = temp
      self.size += 1

  def imprimir (self):
    if self.head is None:
      return
    node = self.head
    print(node.id, end = " => ")
    while node.next:
      node = node.next
      print(node.id, end = " => ")

  def eliminar (self, id):
    node = self.head

    while node is not None:
      if node.id == id:
        if node.previous is not None:
          if node.next:
            node.previous.next = node.next
            node.next.previous = node.previous
          else:
            node.previous.next = None
            self.last = node.previous
        else:
          self.head = node.next
          node.next.previous = self.head
        self.size -= 1
        return True
      else:
        node = node.next
    return False

# Nodo Lista Simple Componentes Lineas Produccion
class nodeLSCLP:
    def __init__(self, id=None, next=None, previous=None):
        self.id = id
        self.next = next
        self.previous = previous

#################################################
# LISTA SIMPLE DE PRODUCTOS MAQUINA
#################################################
class linked_list_productos:
    def __init__(self):
        self.head = None

    def insertar(self, id, Nombre, Elaboracion):
        if not self.head:
            self.head = nodeLSP(id=id, Nombre=Nombre, Elaboracion=Elaboracion)
            return
        current = self.head
        while current.next:
            current = current.next
        current.next = nodeLSP(id=id, Nombre=Nombre, Elaboracion=Elaboracion)

    def mostrar(self):
        actual = self.head
        while actual != None:
            print(actual.id, end="=>")
            actual = actual.next

    def buscar(self, Nombre):
        actual = self.head

        while actual != None:
            if actual.Nombre == Nombre:
                return actual
            actual = actual.next
        return False

    def eliminar(self, id):
        current = self.head
        previous = None

        while current and current.id != id:
            previous = current
            current = current.next

            if previous is None:
                self.head = current.next
            elif current:
                previous.next = current.next
                current.next = None

    def imprimir(self):
        if self.head is None:
            return
        Actual = self.head
        print(Actual.id, end=") ")
        print(Actual.Nombre)
        while Actual.next != None:
            Actual = Actual.next
            print(Actual.id, end=") ")
            print(Actual.Nombre)

# Nodo Lista Simple Productos
class nodeLSP:
    def __init__(self, id=None, Nombre=None, Elaboracion=None, next=None):
        self.id = id
        self.Nombre = Nombre
        self.Elaboracion = Elaboracion
        self.next = next

#################################################
# LISTA SIMPLE COLA DE SECUENCIA PASOS
#################################################
class linked_list_pasos_graphviz:
    def __init__(self):
        self.head = None

    def insertar(self, id, PasoReal, LineaProduccion, Componente):
        if not self.head:
            self.head = nodeLSPG(id=id, PasoReal=PasoReal, LineaProduccion=LineaProduccion, Componente=Componente)
            return
        current = self.head
        while current.next:
            current = current.next
        current.next = nodeLSPG(id=id, PasoReal=PasoReal, LineaProduccion=LineaProduccion, Componente=Componente)

    def mostrar(self):
        actual = self.head
        while actual != None:
            print(actual.Nombre, end="=>")
            actual = actual.next

    def buscar(self, id):
        actual = self.head

        while actual != None:
            if actual.id == id:
                return actual
            actual = actual.next
        return False

    def eliminar(self, id):
        current = self.head
        previous = None

        while current and current.id != id:
            previous = current
            current = current.next

            if previous is None:
                self.head = current.next
            elif current:
                previous.next = current.next
                current.next = None

    def imprimir(self):
        if self.head is None:
            return
        Actual = self.head
        print(Actual.id, end=") ")
        print(Actual.PasoReal + ' Linea: ' + str(Actual.LineaProduccion) + ' Componente: ' + str(Actual.Componente))
        while Actual.next != None:
            Actual = Actual.next
            print(Actual.id, end=") ")
            print(Actual.PasoReal + ' Linea: ' + str(Actual.LineaProduccion) + ' Componente: ' + str(Actual.Componente))

# Nodo Lista Simple Pasos GraphViz
class nodeLSPG:
    def __init__(self, id=None, PasoReal=None, LineaProduccion=None, Componente=None, next=None):
        self.id = id
        self.PasoReal = PasoReal
        self.LineaProduccion = LineaProduccion
        self.Componente = Componente
        self.next = next

#################################################
# LISTA SIMPLE DE SIMULACIONES
#################################################
class linked_list_simulaciones:
    def __init__(self):
        self.head = None

    def insertar(self, id, Nombre, ListaProductos):
        if not self.head:
            self.head = nodeLSS(id=id, Nombre=Nombre, ListaProductos=ListaProductos)
            return
        current = self.head
        while current.next:
            current = current.next
        current.next = nodeLSS(id=id, Nombre=Nombre, ListaProductos=ListaProductos)

    def mostrar(self):
        actual = self.head
        while actual != None:
            print(actual.id, end="=>")
            actual = actual.next

    def buscar(self, Nombre):
        actual = self.head

        while actual != None:
            if actual.Nombre == Nombre:
                return actual
            actual = actual.next
        return False

    def eliminar(self, id):
        current = self.head
        previous = None

        while current and current.id != id:
            previous = current
            current = current.next

            if previous is None:
                self.head = current.next
            elif current:
                previous.next = current.next
                current.next = None

    def imprimir(self):
        if self.head is None:
            return
        Actual = self.head
        print(Actual.id, end=") ")
        print(Actual.Nombre)
        while Actual.next != None:
            Actual = Actual.next
            print(Actual.id, end=") ")
            print(Actual.Nombre)

# Nodo Lista Simple Simulaciones
class nodeLSS:
    def __init__(self, id=None, Nombre=None, ListaProductos=None, next=None):
        self.id = id
        self.Nombre = Nombre
        self.ListaProductos = ListaProductos
        self.next = next

#################################################
# LISTA SIMPLE DE PRODUCTOS SIMULACIONES
#################################################
class linked_list_productos_simulaciones:
    def __init__(self):
        self.head = None

    def insertar(self, id, Nombre):
        if not self.head:
            self.head = nodeLSPS(id=id, Nombre=Nombre)
            return
        current = self.head
        while current.next:
            current = current.next
        current.next = nodeLSPS(id=id, Nombre=Nombre)

    def mostrar(self):
        actual = self.head
        while actual != None:
            print(actual.id, end="=>")
            actual = actual.next

    def buscar(self, Nombre):
        actual = self.head

        while actual != None:
            if actual.Nombre == Nombre:
                return actual
            actual = actual.next
        return False

    def eliminar(self, id):
        current = self.head
        previous = None

        while current and current.id != id:
            previous = current
            current = current.next

            if previous is None:
                self.head = current.next
            elif current:
                previous.next = current.next
                current.next = None

    def imprimir(self):
        if self.head is None:
            return
        Actual = self.head
        print(Actual.id, end=") ")
        print(Actual.Nombre)
        while Actual.next != None:
            Actual = Actual.next
            print(Actual.id, end=") ")
            print(Actual.Nombre)

# Nodo Lista Simple Productos Simulaciones
class nodeLSPS:
    def __init__(self, id=None, Nombre=None, next=None):
        self.id = id
        self.Nombre = Nombre
        self.next = next

##########################################
#Variables Globales                      #
##########################################

#Datos Importantes de el Archivo de Maquina
#Cantidad de Lineas de Produccion
CantidadLineasProduccionMaquina = ''

#Lista de Lineas de Produccion
ListaLineasProduccion = linked_list_lineas_produccion()

#Lista de Productos
ListaProductosMaquina = linked_list_productos()

#Datos Importantes del Archivo de Simulacion
#Lista de Simulaciones
ListaSimulaciones = linked_list_simulaciones()
idSimulacion = 0

#Objetos GUI para usar en los Métodos
ProductoAElegir = ttk.Combobox

#####################################################
#METODOS AYUDA                                      #
#####################################################
# MostrarDatosEstudiante
def MostrarDatosEstudiante():
    tk.messagebox.showinfo(title="Datos Estudiante", message=" Nombre: Bryan Steve Montepeque Santos \n Carnet: 201700375 \n DPI: 2997077170101 \n Curso: IPC2 Sección E")

#####################################################
#CREAR FRAME INICIAL                                #
#####################################################
Ventana = tk.Tk()
Ventana.geometry("900x420")
Ventana.config(bg='#4682B4')
Ventana.title("IPC2_Proyecto2_201700375")

def nada():
    x = 0

#####################################################
#METODOS CARGAR ARCHIVO                             #
#####################################################

#Cargar Archivo Maquina
def CargarArchivoMaquina():
    filters = (("Archivos de XML", "*.xml"), ("Todos los Archivos", "*.*"))
    ArchivoAbierto = tk.filedialog.askopenfilename(filetypes=filters)
    Archivo = ElementTree.parse(ArchivoAbierto)

    #Variables Globales
    global ListaLineasProduccion
    global CantidadLineasProduccionMaquina

    #Analizar el XML

    #Titulo de Inicio
    print('')
    print('---------------------------------------------------')
    print('Inicio de Analisis del Archivo de Máquina')
    print('---------------------------------------------------')
    print('')

    #CantidadLineasProduccion
    CantidadLineasProduccion = Archivo.find('CantidadLineasProduccion').text
    print("Cantidad de Lineas de Produccion: " + str(CantidadLineasProduccion))
    print('')

    CantidadLineasProduccionMaquina = int(CantidadLineasProduccion)
    print('Cantidad de Lineas de Produccion Almacenada')
    print('')

    #ListadoLineasProduccion
    ListadoLineasProduccion = Archivo.find('ListadoLineasProduccion')

    #LineaProduccion
    LineasProduccion = ListadoLineasProduccion.findall('LineaProduccion')

    #Titulo Lista de Lineas de Produccion
    print("Lista de Lineas de Producción: ")
    print('')

    for LineaProduccion in LineasProduccion:

        #Numero de Linea de Produccion
        NumeroLineaProduccion = int(LineaProduccion.find('Numero').text)
        print("Linea de Producción Número: " + str(NumeroLineaProduccion))

        #Cantidad de Componentes
        CantidadComponentes = int(LineaProduccion.find('CantidadComponentes').text)
        print("Cantidad de Componentes: " + str(CantidadComponentes))

        #TiempoEnsamblaje
        TiempoEnsamblaje = int(LineaProduccion.find('TiempoEnsamblaje').text)
        print("Tiempo de Ensamblaje: " + str(TiempoEnsamblaje))

        #Generar Lista de Componentes
        ListaComponentesLineaProduccion = linked_list_de_componentes_lineas_produccion()

        #Ingresar Componentes a la Lista
        for i in range(1, (CantidadComponentes+1)):
            ListaComponentesLineaProduccion.insertarFinal(i)

        #Revisar que si meta bien los Componentes
        print('Estos son los componentes de la Linea de Produccion ' + str(NumeroLineaProduccion) + ': ')
        ListaComponentesLineaProduccion.imprimir()

        #Almacenar La Lista de Linea de Produccion
        ListaLineasProduccion.insertar(NumeroLineaProduccion, CantidadComponentes, TiempoEnsamblaje, ListaComponentesLineaProduccion)
        print("Linea de Produccion Almacenada en Lista ;v")

        # Un Salto de Linea por orden
        print('')

    print('')

    #Variables Globales
    global ListaProductosMaquina

    #ListadoProductos
    ListadoProductos = Archivo.find('ListadoProductos')

    #Productos
    Productos = ListadoProductos.findall('Producto')

    #Titulo de Lista de Productos de Máquina
    print("Lista de Productos en la Máquina: ")
    print('')

    # Crear un id de Producto para identificar los productos con las lineas y almacenarlos en la lista
    idProducto = 0

    for Producto in Productos:

        #Aumentar el id de LineaProduccion
        idProducto = idProducto + 1
        print("El id del Producto es: " + str(idProducto))

        #NombreProducto
        NombreProducto = Producto.find('nombre').text
        print("Nombre de Producto: " + str(NombreProducto))

        #Elaboracion
        ElaboracionProducto = Producto.find('elaboracion').text
        print("Elaboracion del Producto: " + str(ElaboracionProducto))

        #Analizar la Elaboracion del Producto con regex

        #Quitar las "p"
        QuitarDeString = r'p'

        ElaboracionSinP = re.sub(QuitarDeString, '', ElaboracionProducto)

        print('')
        print("Elaboracion sin P: ")
        print(ElaboracionSinP)

        # Crear Lista de Pasos de cada Producto
        ListaPasosProducto = linked_list_pasos_graphviz()

        #MiniAnlizadorLexico
        Estado = 'L'
        Lexema = ''
        idPaso = 0

        # Varaibles para Guardar la Linea y Componente de cada paso
        LineaProduccionPaso = ''
        ComponentePaso = ''

        for Caracter in range(len(ElaboracionSinP)):
            CaracterActual = ElaboracionSinP[Caracter]

            try:
                CaracterSiguiente = ElaboracionSinP[Caracter + 1]  # Este es un pequeño "Scout", puede ver valores que den error
            except:
                CaracterSiguiente = ""
            #Encontrar la Primera L
            if Estado == "L":
                if CaracterActual == "L".upper() and CaracterSiguiente.isdigit():
                    Estado = "DigitosDespuesDeL"
                    Lexema = Lexema + CaracterActual
                    print('Este es el Lexema Actual: ' + str(Lexema))

            #Cualquier Numero de Digitos Entre L y C
            elif Estado == "DigitosDespuesDeL":
                if CaracterActual.isdigit() and CaracterSiguiente.isdigit():
                    Lexema = Lexema + CaracterActual
                    LineaProduccionPaso = LineaProduccionPaso + CaracterActual
                    print('Este es el Lexema Actual: ' + str(Lexema))
                elif CaracterActual.isdigit() and CaracterSiguiente == "C":
                    Estado = "C"
                    Lexema = Lexema + CaracterActual
                    LineaProduccionPaso = LineaProduccionPaso + CaracterActual
                    print('Este es el Lexema Actual: ' + str(Lexema))

            #Encontrar la C
            elif Estado == "C":
                if CaracterActual == "C" and CaracterSiguiente.isdigit():
                    Estado = "DigitosDespuesDeC"
                    Lexema = Lexema + CaracterActual
                    print('Este es el Lexema Actual: ' + str(Lexema))

            #Cualquier Numero de Digitos Despues de la C
            elif Estado == "DigitosDespuesDeC":
                if CaracterActual.isdigit() and CaracterSiguiente.isdigit():
                    Lexema = Lexema + CaracterActual
                    ComponentePaso = ComponentePaso + CaracterActual
                    print('Este es el Lexema Actual: ' + str(Lexema))

                elif CaracterActual.isdigit() and CaracterSiguiente.isspace():
                    Estado = "Espacio"
                    Lexema = Lexema + CaracterActual
                    ComponentePaso = ComponentePaso + CaracterActual
                    idPaso = idPaso + 1

                    # Ingresar Pasos a La Lista de Pasos
                    ListaPasosProducto.insertar(idPaso, Lexema, int(LineaProduccionPaso), int(ComponentePaso))
                    print('El Lexema almacenado es: ' + str(Lexema))
                    print('')

                    #Limpiar las variables del paso
                    LineaProduccionPaso = ''
                    ComponentePaso = ''

                elif CaracterActual.isdigit() and CaracterSiguiente == '':
                    Estado = "Final"
                    Lexema = Lexema + CaracterActual
                    ComponentePaso = ComponentePaso + CaracterActual
                    idPaso = idPaso + 1

                    #Ingresar Pasos a La Lista de Pasos
                    ListaPasosProducto.insertar(idPaso, Lexema, int(LineaProduccionPaso), int(ComponentePaso))
                    print('El Lexema almacenado es: ' + str(Lexema))
                    print('')

                    # Limpiar las variables del paso
                    LineaProduccionPaso = ''
                    ComponentePaso = ''

            elif Estado == "Espacio":
                Estado = "L"
                Lexema = ''
                print('Este es el Lexema Actual: ' + str(Lexema))

        print("Analisis Terminado ;v")

        print('')

        #Prueba que si se hayan ingresado
        print("Estos son los pasos que se guardaron en la Lista de Pasos Reales: ")
        ListaPasosProducto.imprimir()

        # Almacenar La Lista de Productos
        ListaProductosMaquina.insertar(idProducto, NombreProducto, ListaPasosProducto)
        print("Linea de Produccion Almacenada en Lista ;v")

        #Un Salto de Linea por orden
        print('')

    print('')

#Cargar Archivo Simulacion
def CargarArchivoSimulacion():
    filters = (("Archivos de XML", "*.xml"), ("Todos los Archivos", "*.*"))
    ArchivoAbierto = tk.filedialog.askopenfilename(filetypes=filters)
    Archivo = ElementTree.parse(ArchivoAbierto)

    #Variables Globales
    global ListaSimulaciones
    global idSimulacion
    #Analizar el XML

    #Aumentar el id
    idSimulacion = idSimulacion + 1

    # Titulo de Inicio
    print('')
    print('---------------------------------------------------')
    print('Inicio de Analisis del Archivo de Simulación')
    print('---------------------------------------------------')
    print('')

    #Nombre Simulacion
    NombreSimulacion = Archivo.find('Nombre').text
    print('El Nombre de la Simulación Cargada es: ' + str(NombreSimulacion))
    print('')

    #Listado de Productos
    ListadoProductos = Archivo.find('ListadoProductos')

    #Productos
    Productos = ListadoProductos.findall('Producto')

    #Crear Lista de Productos por Simulacion
    ListaProductosSimulacion = linked_list_productos_simulaciones()

    #Crear Lista para agregar los nombres de los productos al ComboBox
    ProductosComboBox = []

    idProductoSimulacion = 0
    for Producto in Productos:

        #Aumentar el id
        idProductoSimulacion = idProductoSimulacion + 1

        #Obtener el Nombre
        NombreProducto = Producto.text
        print('El Nombre del Producto es: ' + str(NombreProducto))
        print('')

        #Almacenar el Producto en una Lista para el ComboBox
        ProductosComboBox.append(NombreProducto)
        print('Estos son los productos en el ComboBox: ')
        print(ProductosComboBox)

        #Guardar el Producto enla Lista de Ssimulacion
        ListaProductosSimulacion.insertar(idProductoSimulacion, NombreProducto)
        print('Producto Almacenado en la Lista de Productos de Simulación')
        print('')

    #Guardar La Simulacion en la lista de Simulaciones
    ListaSimulaciones.insertar(idSimulacion, NombreSimulacion, ListaProductosSimulacion)
    print('Simulación Almacenada en la Lista de Simulaciones')
    print('')

    #Agregar los Nombres de los productos al ComboBox:
    ProductoAElegir['values'] = ProductosComboBox

    #Mensaje Analisis Terminado
    print('Analisis Terminado ;v')
    print('')

#####################################################
#METODOS PROCESAR ARCHIVO                           #
#####################################################
def RealizarSimulacion():

    #Revisar que Ambos Archivos esten cargados:
    if ListaLineasProduccion.head == None:
        print('No se ha cargado el Archivo de Maquina >:v')
        tk.messagebox.showinfo(title="Info", message="No se ha cargado el Archivo de Maquina >:v")
    elif ListaSimulaciones.head == None:
        print('No se ha cargado el Archivo de Simulaciones >:v')
        tk.messagebox.showinfo(title="Info", message="No se ha cargado el Archivo de Simulaciones >:v")
    else:

        #Obtener el nombre seleccionado del ComboBox y buscar el nombre en la lista de productos
        if ProductoAElegir.get() == '':
            print("No ha seleccionado un producto >:v")
            tk.messagebox.showinfo(title="Info", message="No ha seleccionado un producto >:v")
        else:
            ProductoComboBox = ProductoAElegir.get()
            ProductoSeleccionado = ListaProductosMaquina.buscar(ProductoComboBox)

            #Obtener la lista de Pasos para el Producto
            PasosProducto = ProductoSeleccionado.Elaboracion

            #Ver que sí tenga la lista correcta
            print('Estos son los pasos de ' + ProductoComboBox +': ')
            PasosProducto.imprimir()
            print('')

            #Cambiar El Label de Pasos Necesarios:
            ActualPasosNecesarios = PasosProducto.head
            StringPasosNecesarios = ''
            while ActualPasosNecesarios != None:
                StringPasosNecesarios = StringPasosNecesarios + str(ActualPasosNecesarios.PasoReal) + '\n'
                ActualPasosNecesarios = ActualPasosNecesarios.next

            #Poner el String en el Label
            textPasosNReal.delete('1.0', 'end')
            textPasosNReal.insert(INSERT, StringPasosNecesarios)

            # Generar Archivo HTML
            ####################################################################

            #Empezar el String del HTML
            StringDocumentoHTML ='<!DOCTYPE html>\n<html>\n  <head>\n<style>\n table, th, td {\n border: 1px solid black;\n}\n</style>\n<title>Simulacion HTML</title>\n   </head>\n   <body>\n '

            #Titulo Archivo
            StringDocumentoHTML = StringDocumentoHTML + '<h1>IPC2 Proyecto 2</h1>\n<h2>Tabla de Pasos: </h2>\n'

            # Crear Tabla de Pasos
            StringDocumentoHTML = StringDocumentoHTML + '<table>\n  <tr>\n  <th bgcolor = #A93226>Tiempo (t)</th>\n\n'

            #Recorrer la Lista de Lineas de Produccion

            #Generar Encabezados
            ActualListaLineasProduccion = ListaLineasProduccion.head

            while ActualListaLineasProduccion != None:
                StringDocumentoHTML = StringDocumentoHTML + '<th bgcolor = #F1C40F> Linea de Ensamblaje '+ str(ActualListaLineasProduccion.id) + '</th>\n\n'
                ActualListaLineasProduccion = ActualListaLineasProduccion.next

            #Cerrar los encabezados de la tabla
            StringDocumentoHTML = StringDocumentoHTML + '</tr>\n'

            #Llenar los pasos en la Tabla
            #Recorrer Cada Paso
            ActualPasosProducto = PasosProducto.head

            #SegundoActual
            SegundoActual = 0
            while ActualPasosProducto != None:

                #Recorrer cada Linea de Produccion
                for i in range(1,CantidadLineasProduccionMaquina+1):
                    ActualRecorrerListaLineasProduccion = ListaLineasProduccion.buscar(i)

                    if int(ActualRecorrerListaLineasProduccion.id) == ActualPasosProducto.LineaProduccion:

                        #Obtener el Valor Inicial del Componente Actual
                        ActualComponentesLineaProduccionSeleccionada = ActualRecorrerListaLineasProduccion.Componentes.head

                        if int(ActualComponentesLineaProduccionSeleccionada.id) < ActualPasosProducto.Componente:
                            while ActualComponentesLineaProduccionSeleccionada.id != ActualPasosProducto.Componente:
                                # Empezar la Fila
                                StringDocumentoHTML = StringDocumentoHTML + '<tr>\n'
                                # Agregar Segundo
                                SegundoActual = SegundoActual + 1
                                StringDocumentoHTML = StringDocumentoHTML + '<td bgcolor = #A93226 >' + str(SegundoActual) + '</td>\n'
                                for j in range(1, CantidadLineasProduccionMaquina + 1):
                                    ActualRecorrerListaLineasProduccionWhile = ListaLineasProduccion.buscar(j).id
                                    if ActualRecorrerListaLineasProduccionWhile == ActualPasosProducto.LineaProduccion:
                                        StringDocumentoHTML = StringDocumentoHTML + '<td bgcolor = #2980B9> Mover Brazo, Componente ' + str(ActualComponentesLineaProduccionSeleccionada.id) + '</td>\n'
                                    elif ActualRecorrerListaLineasProduccionWhile != ActualPasosProducto.LineaProduccion:
                                        StringDocumentoHTML = StringDocumentoHTML + '<td bgcolor = #2980B9> No hacer nada </td>\n'
                                # Cerrar la Fila
                                StringDocumentoHTML = StringDocumentoHTML + '</tr>\n\n'
                                ActualComponentesLineaProduccionSeleccionada = ActualComponentesLineaProduccionSeleccionada.next
                            #Ensamblar Componente
                            if ActualComponentesLineaProduccionSeleccionada.id == ActualPasosProducto.Componente:
                                # Empezar la Fila
                                StringDocumentoHTML = StringDocumentoHTML + '<tr>\n'
                                # Agregar Segundo
                                SegundoActual = SegundoActual + 1
                                StringDocumentoHTML = StringDocumentoHTML + '<td bgcolor = #A93226 >' + str(SegundoActual) + '</td>\n'

                                for j in range(1, CantidadLineasProduccionMaquina + 1):
                                    ActualRecorrerListaLineasProduccionWhile = ListaLineasProduccion.buscar(j).id
                                    if ActualRecorrerListaLineasProduccionWhile == ActualPasosProducto.LineaProduccion:
                                        StringDocumentoHTML = StringDocumentoHTML + '<td bgcolor = #2980B9> Ensamblar Componente ' + str(ActualComponentesLineaProduccionSeleccionada.id) + '</td>\n'
                                    elif ActualRecorrerListaLineasProduccionWhile != ActualPasosProducto.LineaProduccion:
                                        StringDocumentoHTML = StringDocumentoHTML + '<td bgcolor = #2980B9> No hacer nada </td>\n'
                                # Cerrar la Fila
                                StringDocumentoHTML = StringDocumentoHTML + '</tr>\n\n'

                        elif int(ActualComponentesLineaProduccionSeleccionada.id) > ActualPasosProducto.Componente:
                            while ActualComponentesLineaProduccionSeleccionada.id != ActualPasosProducto.Componente:
                                # Empezar la Fila
                                StringDocumentoHTML = StringDocumentoHTML + '<tr>\n'
                                # Agregar Segundo
                                SegundoActual = SegundoActual + 1
                                StringDocumentoHTML = StringDocumentoHTML + '<td bgcolor = #A93226 >' + str(SegundoActual) + '</td>\n'
                                for i in range(1, CantidadLineasProduccionMaquina + 1):
                                    ActualRecorrerListaLineasProduccionWhile = ListaLineasProduccion.buscar(j).id
                                    if ActualRecorrerListaLineasProduccionWhile == ActualPasosProducto.LineaProduccion:
                                        StringDocumentoHTML = StringDocumentoHTML + '<td bgcolor = #2980B9> Mover Brazo, Componente ' + str(ActualComponentesLineaProduccionSeleccionada.id) + '</td>\n'
                                    elif ActualRecorrerListaLineasProduccionWhile != ActualPasosProducto.LineaProduccion:
                                        StringDocumentoHTML = StringDocumentoHTML + '<td bgcolor = #2980B9> No hacer nada </td>\n'
                                ActualComponentesLineaProduccionSeleccionada = ActualComponentesLineaProduccionSeleccionada.previous
                                # Cerrar la Fila
                                StringDocumentoHTML = StringDocumentoHTML + '</tr>\n\n'

                            #Ensamblar Componente
                            if ActualComponentesLineaProduccionSeleccionada.id == ActualPasosProducto.Componente:
                                # Empezar la Fila
                                StringDocumentoHTML = StringDocumentoHTML + '<tr>\n'
                                # Agregar Segundo
                                SegundoActual = SegundoActual + 1
                                StringDocumentoHTML = StringDocumentoHTML + '<td bgcolor = #A93226 >' + str(
                                    SegundoActual) + '</td>\n'

                                for j in range(1, CantidadLineasProduccionMaquina + 1):
                                    ActualRecorrerListaLineasProduccionWhile = ListaLineasProduccion.buscar(j).id
                                    if ActualRecorrerListaLineasProduccionWhile == ActualPasosProducto.LineaProduccion:
                                        StringDocumentoHTML = StringDocumentoHTML + '<td bgcolor = #2980B9> Ensamblar Componente ' + str(
                                            ActualComponentesLineaProduccionSeleccionada.id) + '</td>\n'
                                    elif ActualRecorrerListaLineasProduccionWhile != ActualPasosProducto.LineaProduccion:
                                        StringDocumentoHTML = StringDocumentoHTML + '<td bgcolor = #2980B9> No hacer nada </td>\n'
                                # Cerrar la Fila
                                StringDocumentoHTML = StringDocumentoHTML + '</tr>\n\n'

                #Avanzar el Actual de Pasos
                ActualPasosProducto = ActualPasosProducto.next

            # Generar el HTML
            with open("TablaPasos.html", "w") as ArchivoHTML:
                # Terminar el String del HTML
                StringDocumentoHTML = StringDocumentoHTML + '</tr>\n</table>\n</body>\n</html>'
                ArchivoHTML.write(StringDocumentoHTML)
                ArchivoHTML.close()
                print('')
                print("Ahorita te abro el HTML ;v")
                os.system("start TablaPasos.html")

            #Generar Archivo de GraphViz
            ####################################################################
            ActualGraphviz = PasosProducto.head

            #Empezar el String de Graphviz
            CodigoGraphViz = 'digraph D {\n rankdir = LR;\n'
            idNodoGraphViz = 1
            while ActualGraphviz != None:
                #Agregar Nodo por Nodo
                CodigoGraphViz = CodigoGraphViz + str(idNodoGraphViz) + '[label=\"' + ActualGraphviz.PasoReal + '\", shape=box];\n'

                # Aumentar el numero del apuntador
                idNodoGraphViz = idNodoGraphViz + 1

                #Agregar Los Apuntadores
                if ActualGraphviz.next != None:
                    CodigoGraphViz = CodigoGraphViz + str(idNodoGraphViz - 1) + '->' + str(idNodoGraphViz) + ';\n'
                elif ActualGraphviz.next == None:
                    CodigoGraphViz = CodigoGraphViz

                ActualGraphviz = ActualGraphviz.next

            #Generar El Archivo de GraphViz:
            with open("SecuenciaPasos.dot", 'w') as ArchivoGraphViz:

                # Agregar el Nombre del Producto
                CodigoGraphViz = CodigoGraphViz + "\nlabelloc = \"t\";\nlabel=\"" + ProductoComboBox + "\";\n"

                # TerminarElStringDelCodigo
                CodigoGraphViz = CodigoGraphViz + "\n}"

                ArchivoGraphViz.write(CodigoGraphViz)
            ArchivoGraphViz.close()
            print('')
            print('Ahorita te abro el Grafo ;v')
            os.system("dot -Tjpg SecuenciaPasos.dot -o SecuenciaPasosReal.jpg")
            os.system("start SecuenciaPasosReal.jpg")

def RealizarSimulacionTiempo():

    #Revisar que Ambos Archivos esten cargados:
    if ListaLineasProduccion.head == None:
        print('No se ha cargado el Archivo de Maquina >:v')
        tk.messagebox.showinfo(title="Info", message="No se ha cargado el Archivo de Maquina >:v")
    elif ListaSimulaciones.head == None:
        print('No se ha cargado el Archivo de Simulaciones >:v')
        tk.messagebox.showinfo(title="Info", message="No se ha cargado el Archivo de Simulaciones >:v")
    else:

        #Obtener el nombre seleccionado del ComboBox y buscar el nombre en la lista de productos
        if ProductoAElegir.get() == '':
            print("No ha seleccionado un producto >:v")
            tk.messagebox.showinfo(title="Info", message="No ha seleccionado un producto >:v")
        else:
            ProductoComboBox = ProductoAElegir.get()
            ProductoSeleccionado = ListaProductosMaquina.buscar(ProductoComboBox)

            #Obtener la lista de Pasos para el Producto
            PasosProducto = ProductoSeleccionado.Elaboracion

            #Ver que sí tenga la lista correcta
            print('Estos son los pasos de ' + ProductoComboBox +': ')
            PasosProducto.imprimir()
            print('')

            #Cambiar El Label de Pasos Necesarios:
            ActualPasosNecesarios = PasosProducto.head
            StringPasosNecesarios = ''
            while ActualPasosNecesarios != None:
                StringPasosNecesarios = StringPasosNecesarios + str(ActualPasosNecesarios.PasoReal) + '\n'
                ActualPasosNecesarios = ActualPasosNecesarios.next

            #Poner el String en el Label
            textPasosNReal.delete('1.0', 'end')
            textPasosNReal.insert(INSERT, StringPasosNecesarios)

            #Obtener el Valor de t
            Segundo = int(txtbbxMomentoProcesoActual.get())

            #Generar Archivo de GraphViz
            ActualGraphviz = PasosProducto.head

            #Empezar el String de Graphviz
            CodigoGraphViz = 'digraph D {\n rankdir = LR;\n'
            idNodoGraphViz = 1

            for i in range(1, (Segundo+1)):
                if PasosProducto.buscar(i) != False:
                    #Agregar Nodo por Nodo
                    CodigoGraphViz = CodigoGraphViz + str(idNodoGraphViz) + '[label=\"' + PasosProducto.buscar(i).PasoReal + '\", shape=box];\n'

                    # Aumentar el numero del apuntador
                    idNodoGraphViz = idNodoGraphViz + 1

                    # Agregar Los Apuntadores
                    if i < Segundo:
                        CodigoGraphViz = CodigoGraphViz + str(idNodoGraphViz - 1) + '->' + str(idNodoGraphViz) + ';\n'
                    elif i == Segundo:
                        CodigoGraphViz = CodigoGraphViz
                else:
                    CodigoGraphViz = CodigoGraphViz

            #Generar El Archivo de GraphViz:
            with open("SecuenciaPasos.dot", 'w') as ArchivoGraphViz:

                # Agregar el Nombre del Producto
                CodigoGraphViz = CodigoGraphViz + "\nlabelloc = \"t\";\nlabel=\"" + ProductoComboBox + "\";\n"

                # TerminarElStringDelCodigo
                CodigoGraphViz = CodigoGraphViz + "\n}"

                ArchivoGraphViz.write(CodigoGraphViz)
            ArchivoGraphViz.close()
            print('')
            print('Ahorita te lo abro ;v')
            os.system("dot -Tjpg SecuenciaPasos.dot -o SecuenciaPasosReal.jpg")
            os.system("start SecuenciaPasosReal.jpg")

# MenuBar
BarraMenu = Menu(Ventana)

# Menu Archivo
MenuArchivo = Menu(BarraMenu, tearoff=0)
MenuArchivo.add_command(label="Cargar Archivo Configuracion Maquina", command=CargarArchivoMaquina)
MenuArchivo.add_command(label="Cargar Archivo Configuracion Simulacion", command=CargarArchivoSimulacion)
MenuArchivo.add_command(label="Salir", command=Ventana.quit)
BarraMenu.add_cascade(label="Archivo", menu=MenuArchivo)

# Menu Ayuda
MenuAyuda = Menu(BarraMenu, tearoff=0)
MenuAyuda.add_command(label="Mostrar Datos Estudiante", command=MostrarDatosEstudiante)
MenuAyuda.add_command(label="Mostrar Documentación", command=nada)
BarraMenu.add_cascade(label="Ayuda", menu=MenuAyuda)

Ventana.config(menu=BarraMenu)

#ComboBox De Productos
lblProductoElegir = Label(Ventana, text="Producto Elegido: ", bg="#00FF7F")
lblProductoElegir.place(x=30, y=10, height=20, width=100)

ProductoAElegir = ttk.Combobox(Ventana, textvariable='ProductoElegido')
ProductoAElegir.place(x=30, y=40, height=20, width=200)

#PasosNecesarios
lblPasosN = Label(Ventana, text="Pasos Necesarios: ", bg="#00FF7F")
lblPasosN.place(x=50, y=70, height=20, width=150)

#Crear Texto
textPasosNReal = Text(Ventana, bg='#8FBC8F')
textPasosNReal.insert(INSERT, "Pasos a Realizar")

#Crear Scrollbar
ScrollbarPasosNReal = ttk.Scrollbar(textPasosNReal, orient='vertical', command=textPasosNReal.yview)
ScrollbarPasosNReal.place(x=250, y=0, height=270)
textPasosNReal.config(yscrollcommand=ScrollbarPasosNReal.set)
textPasosNReal.place(x=50, y=100, height=270, width=270)

#Boton Realizar Simulacion 100%
btnRealizarSimulacion = Button(Ventana, text="Realizar Simulación", command=RealizarSimulacion)
btnRealizarSimulacion.place(x=200, y=380, height=20, width=120)

#ProcesoActual
lblProcesoActual = Label(Ventana, text="Proceso en el momento: ", bg="#00FF7F")
lblProcesoActual.place(x=350, y=10, height=20, width=170)

txtbbxMomentoProcesoActual = Entry(Ventana)
txtbbxMomentoProcesoActual.place(x=530, y=10, height=20, width=50)

btnMostrarProceso = Button(Ventana, text="Mostrar Proceso", command=RealizarSimulacionTiempo)
btnMostrarProceso.place(x=590, y=10, height=20, width=120)

#Etiqeutas Nombre Matrices
Ventana.mainloop()

Semestre 4/IPC2/Laboratorio/Proyectos/Oportunidad 3/Proyecto 2/Entregables/CODIGO/main.py


from flask import Flask, request, url_for, jsonify
from includes.database import *
app = Flask(__name__)

session = Session(engine)

# Function where we handle validation
def validationCheck(**kwargs):
	# validation - if these values fail return a 400 (BAD REQUEST) error
	content = []
	if kwargs.get('grade') not in ['A', 'B', 'C']:
		content.append({'grade': kwargs.get('grade') + ' is not a valid grade.  Only A, B, or C are acceptable'})

	# returning there is content return True else return response
	if len(content) > 0:
		resp = jsonify(content)
		resp.status_code = 400
		return resp
	else:
		return True

# Request for getting our data
@app.route("/getdata", methods=['GET'])
def getData():
	grade = 'B' if request.args.get('grade') is None else request.args.get('grade')
	cuisine_type = 'Thai' if request.args.get('cuisine_type') is None else request.args.get('cuisine_type')

	# validation - if these values fail return a 400 (BAD REQUEST) error
	resp = validationCheck(grade=grade)
	if resp is not True:
		return resp

	# validation passed.  We can continue

	# Query that retrieves call
	query = """
		SELECT r.camis_id AS camis,
			   r.dba,
			   b.name AS boro,
			   r.building,
			   r.street,
			   r.zip_code AS zipcode,
			   r.phone,
			   g.label AS grade
		FROM inspections i
		INNER JOIN restaurants r ON i.restaurant_id = r.camis_id
		INNER JOIN boros b ON r.boro_id = b.id
		INNER JOIN cuisines c ON r.cuisine_id = c.id
		INNER JOIN actions a ON i.action_id = a.id
		INNER JOIN violation_codes v ON i.violation_id = v.id
		INNER JOIN critical_flags cf ON i.critical_flag_id = cf.id
		INNER JOIN grades g ON i.grade_id = g.id
		INNER JOIN inspection_types it ON i.inspect_type_id = it.id
		WHERE c.id = (
			SELECT c.id FROM cuisines c WHERE LOWER(c.description) = LOWER(:cuisine_type)
		)
		GROUP BY r.camis_id, r.dba, b.name, g.label, g.score
		HAVING g.score >= (
			SELECT g.score FROM grades g WHERE LOWER(g.label) = LOWER(:grade)
		)
		ORDER BY r.dba
	"""

	# executing our query
	qParams = {
		'grade': grade,
		'cuisine_type': cuisine_type
	}
	results = session.execute(query, qParams)

	# outputting results as a json data set
	data = []
	for i, row in enumerate(results):
		# This puts the data in dicionary format so it can outputed in JSON properly
		data.append(dict(zip(row.keys(), row)))

	return jsonify(data)

if __name__ == "__main__":
	app.run()


#!/usr/bin/python3.4
#-*- coding:utf-8 -*-
import numpy as np
from npcvt import nprgb2jch,npvalidrgb
from colorindex import jch_rgb_index,c_valid_index
from PIL import Image

mask_value=[0,255,255]

def image2jch(filename):
    f = Image.open(filename).convert('RGB')
    a = np.array(f)
    f.close()
    return nprgb2jch(a.reshape(-1,3)).reshape(a.shape).astype('int')

def jch2image(jcho):
    jch = jcho.astype('int').reshape(-1,3)
    f = jch_rgb_index[jch[:,0],jch[:,1],jch[:,2]].reshape(jcho.shape)
    return Image.fromarray(f)

def j_lin(j2,center,proportion,outrange=False,replace=[[30,114,280],[50,114,14]]):
    jch = j2.reshape(-1,3)
    jch= (jch-[center,0,0])*[1+proportion,1,1]+[center,0,0]
    if not outrange:
        jch[...,0] = np.where(jch[...,0]<0,0,jch[...,0])
        jch[...,0] = np.where(jch[...,0]>100,100,jch[...,0])
        return jch.reshape(j2.shape).astype('int')
    else:
        j = jch[:,0]
        low_valid = j<0 #np.all(jch<0,1)
        jch[low_valid] = replace[0] #= np.where(low_valid,replace[0],jch)
        high_valid= j>100#np.all(jch>100,1)
        jch[high_valid]= replace[1] #= np.where(high_valid,replace[1],jch)
        return jch.reshape(j2.shape).astype('int')

def c_lin(j2,center,proportion,outrange=False,replace=\
          [[40,114,300],[80,114,140],[90,114,100]]):
    jch = j2.reshape(-1,3)
    jch= ((jch-[0,center,0])*[1,1+proportion,1]+[0,center,0]).astype('int')
    if not outrange:
        jch[...,1] = np.where(jch[...,1]<0,0,jch[...,1])
        jch[...,1] = np.where(jch[...,1]>114,114,jch[...,1])
        return jch.reshape(j2.shape).astype('int')
    else:
        c = jch[:,1]
        low_valid = c<0
        high_valid = c>114
        jch[low_valid] = replace[0]
        jch[high_valid] = replace[1]
        c_valid = np.invert(c_valid_index[jch[:,0],jch[:,1],jch[:,2]])
        jch[c_valid] = replace[2]
        jch[low_valid] = replace[0]
        jch[high_valid] = replace[1]
        return jch.reshape(j2.shape).astype('int')

def distribution(jch,prt="c"):
    prt_dict = {"j":0,"c":1,"h":2}
    #jch = j2.reshape(-1,3)
    seq = jch[...,prt_dict[prt]].astype('uint8')
    return Image.fromarray(seq)

image_path = "/Users/yudan/Documents/mypicworks/samples/"

def tst(name,center,prop,func,**kwargs):
    a = image2jch(image_path+name)
    jch = func(a,center,prop,kwargs)
    return jch2image(jch)

def tst2():
    b = Image.open(image_path+"m.jpg")
    e = Image.open(image_path+"l.jpg")
    f = Image.open(image_path+"k.jpg")
    #b = image2jch(image_path+"b.jpg")
    #e = image2jch(image_path+"e.jpg")
    #f = image2jch(image_path+"f.jpg")
    size = (1000,1000)
    samp = Image.BICUBIC
    s1 = b.resize(size,samp)
    s2 = e.resize(size,samp)
    s3 = f.resize(size,samp)

    # nprgb2jch(a.reshape(-1,3)).reshape(a.shape).astype('int')
    #b = #image2jch(b)
    #e = #image2jch(e)
    #f = #image2jch(f)
    g = np.array
    jch1 = nprgb2jch(g(s1).reshape(-1,3)).reshape(g(s1).shape).astype('int')
    jch2 = nprgb2jch(g(s2).reshape(-1,3)).reshape(g(s2).shape).astype('int')
    jch3 = nprgb2jch(g(s3).reshape(-1,3)).reshape(g(s3).shape).astype('int')
    #jch2 = image2jch(s2)
    #jch3 = image2jch(s3)
    jjj  = np.zeros(jch1.shape,dtype="int")
    jjj[...,0]=jch1[...,0]
    jjj[...,1]=jch2[...,1]
    jjj[...,2]=jch3[...,2]
    return (jch2image(jjj),s1,s2,s3)
#if __name__=="__main__":
#    from


import csv
from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import confusion_matrix
from sklearn.metrics import f1_score

# Loop through each line in the csv and store the details as an object.
def openFile(fileName):
    # Open the given file name.
    with open(fileName) as csv_file:
        csv_reader = csv.reader(csv_file)
        X,y = [],[] # Stores the data and labels.
        for i,row in enumerate(csv_reader):
            # Ignore row 1 because it's the column names.
            if i == 0:
                continue
            # For each row extract the Class ID and F1,F2,F3 values.
            temp_classID = int(row[6])
            temp_F1 = float(row[8])
            temp_F2 = float(row[9])
            temp_F3 = float(row[10])
            # Append new entry.
            X.append([temp_F1,temp_F2,temp_F3])
            y.append(temp_classID)
    # Shuffle entries and split them into data (X) and labels (y).
    return X,y

def buildConfusionMatrix(y_true,y_pred):
    print("\nThe confusion matrix is:")
    print(confusion_matrix(y_true,y_pred,labels=[1, 2, 3]))
    # Micro - calculates by counting total true positives, false negatives and false positives.
    print("F1 score:" + str(round(f1_score(y_true,y_pred, average='micro'),4)))

def doKNN(X,y,percent,rand,k,metric):
    # Split the data into training and testing set.
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=percent, random_state=rand)
    # Create a KNN classifier with k neighbours and a given metric.
    knn = KNeighborsClassifier(n_neighbors=k, metric=metric)
    # Train the model by passing in our training data to fit our model to the training data.
    knn.fit(X_train, y_train)
    # Make predictions on the test data.
    y_pred = knn.predict(X_test)
    # Build a confusion matrix and displays the F1 score using the predicted and actual labels.
    buildConfusionMatrix(y_test,y_pred)

def runner():
    # Open file and obtain data and labels.
    X,y = openFile("Data.csv")
    # Customizable values for the KNN classifier and splitting percentage.
    percentage = 0.25
    k = 5
    metric = "manhattan"
    # Does KNN 5 times for different training/testing sets by changing the random seed generator.
    for i in range(5):
        doKNN(X, y, percentage, i, k, metric)
runner()

task 3 - speech phoneme analysis and classification/code/main.py


import torch
import sys,os
import random
sys.path.insert(0, os.path.join(sys.path[0], '..'))
from collections import OrderedDict

import numpy as np
from utils.import_modules import import_attr
from models.vae_models import Autoencoder, VariationalAutoencoder, CondVariationalAutoencoder
from utils.load_save import save_dictionary, load_dictionary
from torch.optim.lr_scheduler import MultiStepLR
from examples.config_file import config

ig = config()

def train(autoencoder, data, labels=None, ig=None):
    opt = torch.optim.Adam(autoencoder.parameters(), lr=ig.lr)
    scheduler = MultiStepLR(opt, milestones=[1700,1900], gamma=0.1)
    for epoch in range(ig.n_epochs):
        total_loss = 0
        for i in range(0,len(data),ig.bs):
            x = torch.Tensor(data[i:i+ig.bs]).to(ig.device) # GPU
            opt.zero_grad()
            if ig.ea_type == 'cvae':
               y = torch.Tensor(labels[i:i+ig.bs]).to(ig.device)
               x_hat = autoencoder(x, y/5100.)
            else:
               x_hat = autoencoder(x)
            if ig.ea_type == 'ae':
               loss = ((x - x_hat)**2).sum()
            else:
               loss = ((x - x_hat)**2).sum() + autoencoder.encoder.kl
            loss.backward()
            opt.step()
            total_loss += loss.item()
        print("Epoch:{}/{}, Loss:{}".format(epoch,ig.n_epochs,total_loss/len(data)))
        if epoch % 100 == 0:
           torch.save(autoencoder.state_dict(), ig.model_checkpoint + ig.ea_type+'_encoder_in'+str(ig.high_dim)+'_'+'proj'+str(ig.low_dim)+'.pth')
           torch.save(autoencoder.state_dict(), ig.model_checkpoint + ig.ea_type+'_decoder_in'+str(ig.high_dim)+'_'+'proj'+str(ig.low_dim)+'.pth')
#        scheduler.step(epoch)
    return autoencoder

def count_parameters(model):
    return sum(p.numel() for p in model.parameters() if p.requires_grad)

Xtrain = np.zeros([0, ig.high_dim])
Ytrain = np.zeros([0, 1])
arr = os.listdir(ig.data_samples_dir)
threshold = 100.
k = 0
for i in arr:
    if 'turbo' in i and 'no' in i and k < 5:
       dict_input = load_dictionary(ig.data_samples_dir+i)
       print(i)
       X_ = dict_input['Xepisodes']
       Y_ = dict_input['Yepisodes'].reshape(-1,)
       if Y_[Y_<threshold] != []:
          Xtrain = np.append(Xtrain, X_[(Y_<threshold),:], axis=0)
          Ytrain = np.append(Ytrain, Y_[(Y_<threshold)].reshape(-1,1), axis=0)
       k += 2

if ig.ea_type == 'vae':
   vae = VariationalAutoencoder(ig.high_dim, ig.low_dim).to(ig.device) # GPU
elif ig.ea_type == 'cvae':
   vae = CondVariationalAutoencoder(ig.high_dim, ig.low_dim).to(ig.device) # GPU
elif ig.ea_type == 'ae':
   vae = Autoencoder(ig.high_dim, ig.low_dim).to(ig.device) # GPU

print(Ytrain.shape, count_parameters(vae))
input('enter')

vae = train(vae, Xtrain, Ytrain, ig)
torch.save(vae.encoder.state_dict(), ig.model_checkpoint + ig.ea_type+'_encoder_in'+str(ig.high_dim)+'_'+'proj'+str(ig.low_dim)+'.pth')
torch.save(vae.decoder.state_dict(), ig.model_checkpoint + ig.ea_type+'_decoder_in'+str(ig.high_dim)+'_'+'proj'+str(ig.low_dim)+'.pth')

sample = torch.Tensor(Xtrain[:1]).to(ig.device)
if ig.ea_type == 'vae':
   print(sample.data,vae(sample))
elif ig.ea_type == 'cvae':
   Y = torch.Tensor([[0]])
   print(sample.data,vae(sample, Y))


# 6.0001/6.00 Problem Set 5 - RSS Feed Filter
# Name:
# Collaborators:
# Time:

import feedparser
import string
import time
import threading
from project_util import translate_html
from mtTkinter import *
from datetime import datetime
import pytz

#-----------------------------------------------------------------------

#======================
# Code for retrieving and parsing
# Google and Yahoo News feeds
# Do not change this code
#======================

def process(url):
    """
    Fetches news items from the rss url and parses them.
    Returns a list of NewsStory-s.
    """
    feed = feedparser.parse(url)
    entries = feed.entries
    ret = []
    for entry in entries:
        guid = entry.guid
        title = translate_html(entry.title)
        link = entry.link
        description = translate_html(entry.description)
        pubdate = translate_html(entry.published)

        try:
            pubdate = datetime.strptime(pubdate, "%a, %d %b %Y %H:%M:%S %Z")
            pubdate.replace(tzinfo=pytz.timezone("GMT"))
          #  pubdate = pubdate.astimezone(pytz.timezone('EST'))
          #  pubdate.replace(tzinfo=None)
        except ValueError:
            pubdate = datetime.strptime(pubdate, "%a, %d %b %Y %H:%M:%S %z")

        newsStory = NewsStory(guid, title, description, link, pubdate)
        ret.append(newsStory)
    return ret

#======================
# Data structure design
#======================

# Problem 1

class NewsStory(object):
    def __init__(self, guid, title, description, link, pubdate):
        self.guid = guid
        self.title = title
        self.description = description
        self.link = link
        self.pubdate = pubdate

    def get_guid(self):
        return self.guid

    def get_title(self):
        return self.title

    def get_description(self):
        return self.description

    def get_link(self):
        return self.link

    def get_pubdate(self):
        return self.pubdate

#======================
# Triggers
#======================

class Trigger(object):
    def evaluate(self, story):
        """
        Returns True if an alert should be generated
        for the given news item, or False otherwise.
        """
        # DO NOT CHANGE THIS!
        raise NotImplementedError

# PHRASE TRIGGERS

#Problem 2
class PhraseTrigger(Trigger):
    def __init__(self, phrase):
        self.phrase = phrase

    def is_phrase_in(self, text):
        """
        Returns True if self.phrase is present in text,
        otherwise returns False.
        """

        #Get rid of caps
        text = text.lower()

        #Get rid of punctuation
        for i in string.punctuation:
            text = text.replace(i,' ')

        #Get rid of spacing
        text = ' '+' '.join(text.split())+' '

        #Add spaces to avoid phrase being part of longer word
        phrase = ' '+self.phrase.lower()+' '

        if phrase in text:
            return True

        return False

# Problem 3
class TitleTrigger(PhraseTrigger):
    def evaluate(self, story):
        """
        Returns True if self.phrase is present in text,
        otherwise returns False.
        """
        if self.is_phrase_in(story.get_title()):
            return True

# Problem 4
class DescriptionTrigger(PhraseTrigger):
    def evaluate(self, story):
        """
        Returns True if self.phrase is present in text,
        otherwise returns False.
        """
        if self.is_phrase_in(story.get_description()):
            return True

# TIME TRIGGERS

# Problem 5

class TimeTrigger(Trigger):
    def __init__(self, time):
        self.time = datetime.strptime(time, "%d %b %Y %H:%M:%S")

# Problem 6

class BeforeTrigger(TimeTrigger):
    def evaluate(self, story):
        if story.get_pubdate() < self.time:
            return True

class AfterTrigger(TimeTrigger):
    def evaluate(self, story):
        if story.get_pubdate() > self.time:
            return True

# COMPOSITE TRIGGERS

# Problem 7

class NotTrigger(Trigger):
    def __init__(self, trigger):
        self.trigger = trigger

    def evaluate(self, story):
        if not self.trigger.evaluate(story):
            return True

# Problem 8

class AndTrigger(Trigger):
    def __init__(self, trigger1, trigger2):
        self.trigger1 = trigger1
        self.trigger2 = trigger2

    def evaluate(self, story):
        if self.trigger1.evaluate(story) and self.trigger2.evaluate(story):
            return True

# Problem 9

class OrTrigger(Trigger):
    def __init__(self, trigger1, trigger2):
        AndTrigger.__init__(self, trigger1, trigger2)

    def evaluate(self, story):
        if self.trigger1.evaluate(story) or self.trigger2.evaluate(story):
            return True

#======================
# Filtering
#======================

# Problem 10
def filter_stories(stories, triggerlist):
    """
    Takes in a list of NewsStory instances.

    Returns: a list of only the stories for which a trigger in triggerlist fires.
    """

    list = []

    for story in stories:
        for trigger in triggerlist:
                if trigger.evaluate(story):
                    list.append(story)

    return list

#======================
# User-Specified Triggers
#======================
# Problem 11

def read_trigger_config(filename):
    """
    filename: the name of a trigger configuration file

    Returns: a list of trigger objects specified by the trigger configuration
        file.
    """

    trigger_file = open(filename, 'r')
    lines = []
    for line in trigger_file:
        line = line.rstrip()
        if not (len(line) == 0 or line.startswith('//')):
            lines.append(line)

    triggers = {}
    triggerlist = []

    for line in lines:
        line = line.split(',')

        if line[0] != 'ADD':

                if line[1] == 'TITLE':
                    triggers[line[0]] = TitleTrigger(line[2])

                elif line[1] == 'DESCRIPTION':
                    triggers[line[0]] = DescriptionTrigger(line[2])

                elif line[1] == 'AFTER':
                    triggers[line[0]] = AfterTrigger(line[2])

                elif line[1] == 'BEFORE':
                    triggers[line[0]] = BeforeTrigger(line[2])

                elif line[1] == 'NOT':
                    triggers[line[0]] = NotTrigger(line[2])

                elif line[1] == 'AND':
                    triggers[line[0]] = AndTrigger(triggers.get(line[2]),triggers.get(line[3]))

                elif line[1] == 'OR':
                    triggers[line[0]] = OrTrigger(triggers.get(line[2]),triggers.get(line[3]))

        else:
            for i in line[1:]:
                triggerlist.append(triggers.get(i))

    return triggerlist

SLEEPTIME = 120 #seconds -- how often we poll

def main_thread(master):
    # A sample trigger list - you might need to change the phrases to correspond
    # to what is currently in the news
    try:
        # Problem 11
        triggerlist = read_trigger_config('ps5/triggers.txt')

        # HELPER CODE - you don't need to understand this!
        # Draws the popup window that displays the filtered stories
        # Retrieves and filters the stories from the RSS feeds
        frame = Frame(master)
        frame.pack(side=BOTTOM)
        scrollbar = Scrollbar(master)
        scrollbar.pack(side=RIGHT,fill=Y)

        t = "Google Top News"
        title = StringVar()
        title.set(t)
        ttl = Label(master, textvariable=title, font=("Helvetica", 18))
        ttl.pack(side=TOP)
        cont = Text(master, font=("Helvetica",14), yscrollcommand=scrollbar.set)
        cont.pack(side=BOTTOM)
        cont.tag_config("title", justify='center')
        button = Button(frame, text="Exit", command=root.destroy)
        button.pack(side=BOTTOM)
        guidShown = []
        def get_cont(newstory):
            if newstory.get_guid() not in guidShown:
                cont.insert(END, newstory.get_title()+"\n", "title")
                cont.insert(END, "\n---------------------------------------------------------------\n", "title")
                cont.insert(END, newstory.get_description())
                cont.insert(END, "\n*********************************************************************\n", "title")
                guidShown.append(newstory.get_guid())

        while True:

            print("Polling . . .", end=' ')
            # Get stories from Google's Top Stories RSS news feed
            stories = process("http://news.google.com/news?output=rss")

            # Get stories from Yahoo's Top Stories RSS news feed
            # stories.extend(process("http://news.yahoo.com/rss/topstories"))

            stories = filter_stories(stories, triggerlist)

            list(map(get_cont, stories))
            scrollbar.config(command=cont.yview)

            print("Sleeping...")
            time.sleep(SLEEPTIME)

    except Exception as e:
        print(e)

if __name__ == '__main__':
    root = Tk()
    root.title("Some RSS parser")
    t = threading.Thread(target=main_thread, args=(root,))
    t.start()
    root.mainloop()


import json
from django.http import HttpResponse, JsonResponse
from django.views.generic import DetailView, ListView, CreateView, UpdateView, DeleteView
from django.utils.decorators import method_decorator
from django.views.decorators.csrf import csrf_exempt

from SKYPRO_HOMEWORK_27 import settings
from ads.models import Ad, Category
from django.core.paginator import Paginator
from django.shortcuts import get_object_or_404

from users.models import User

def index(request):
    return HttpResponse(200, {"status": "ok"})

class AdListView(ListView):
    model = Ad

    def get(self, request, *args, **kwargs):
        super().get(self, request, *args, **kwargs)

        all_ads = self.object_list.select_related('author', 'category').order_by('-price')

        paginator = Paginator(all_ads, settings.TOTAL_ON_PAGE)
        page_num = int(request.GET.get('page', 1))
        page_obj = paginator.get_page(page_num)

        ads = []
        for ad in page_obj:
            ads.append({
                "id": ad.id,
                "name": ad.name,
                "author_id": ad.author_id,
                "author": ad.author.username,
                "price": ad.price,
                "description": ad.description,
                "image": ad.image.url if ad.image else None,
                "is_published": ad.is_published,
                "category_id": ad.category_id,
                "category": ad.category.name,
        })
        response = {
            "items": ads,
            "num_pages": paginator.num_pages,
            "total": paginator.count,
        }
        return JsonResponse(response, safe=False, json_dumps_params={'ensure_ascii': False}, status=200)

class AdDetailView(DetailView):
    model = Ad

    def get(self, request, *args, **kwargs):
        try:
            ad = self.get_object()
        except Ad.DoesNotExist:
            return JsonResponse({"error": "Not found"}, status=404)

        return JsonResponse({
                "id": ad.id,
                "name": ad.name,
                "author_id": ad.author_id,
                "author": ad.author.username,
                "price": ad.price,
                "description": ad.description,
                "image": ad.image.url if ad.image else None,
                "is_published": ad.is_published,
                "category_id": ad.category_id,
                "category": ad.category.name,
        })

@method_decorator(csrf_exempt, name='dispatch')
class AdCreateView(CreateView):
    model = Ad
    fields = ["name", "author", "price", "description", "image", "is_published", "category"]

    def post(self, request, *args, **kwargs):
        ad_data = json.loads(request.body)

        author = get_object_or_404(User, id=ad_data["author_id"])
        category = get_object_or_404(Category, id=ad_data["category_id"])

        ad = Ad.objects.create(
            name=ad_data["name"],
            author_id=author.id,
            price=ad_data["price"],
            description=ad_data["description"],
            is_published=ad_data["is_published"],
            category_id=category.id,
        )

        return JsonResponse({
            "id": ad.id,
            "name": ad.name,
            "author_id": ad.author_id,
            "author": ad.author.username,
            "price": ad.price,
            "description": ad.description,
            "image": ad.image.url if ad.image else None,
            "is_published": ad.is_published,
            "category_id": ad.category_id,
            "category": ad.category.name,
        }, status=201)

@method_decorator(csrf_exempt, name='dispatch')
class AdImageView(UpdateView):
    model = Ad
    fields = ["name", "author", "price", "description", "image", "is_published", "category"]

    def post(self, request, *args, **kwargs):
        ad = self.get_object()

        ad.image = request.FILES.get("image", None)
        ad.save()

        return JsonResponse({
            "id": ad.id,
            "name": ad.name,
            "author_id": ad.author_id,
            "price": ad.price,
            "description": ad.description,
            "image": ad.image.url if ad.image else None,
            "is_published": ad.is_published,
            "category_id": ad.category_id,
        })

@method_decorator(csrf_exempt, name='dispatch')
class AdUpdateView(UpdateView):
    model = Ad
    fields = ["name", "author", "price", "description", "image", "is_published", "category"]

    def patch(self, request, *args, **kwargs):
        super().post(request, *args, **kwargs)

        ad_data = json.loads(request.body)
        author = get_object_or_404(User, id=ad_data["author_id"])
        category = get_object_or_404(Category, id=ad_data["category_id"])
        ad = self.object

        ad.name = ad_data["name"]
        ad.author_id = author.id
        ad.price = ad_data["price"]
        ad.description = ad_data["description"]
        ad.is_published = ad_data["is_published"]
        ad.category_id = category.id

        ad.save()

        return JsonResponse({
            "id": ad.id,
            "name": ad.name,
            "author_id": ad.author_id,
            "price": ad.price,
            "description": ad.description,
            "image": ad.image.url if ad.image else None,
            "is_published": ad.is_published,
            "category_id": ad.category_id,
        }, status=201)

@method_decorator(csrf_exempt, name="dispatch")
class AdDeleteView(DeleteView):
    model = Ad
    success_url = '/'

    def delete(self, request, *args, **kwargs):
        super().delete(request, *args, **kwargs)

        return JsonResponse({'status': 'ok'}, status=200)