# Process Data
# Created by Ziyu He

import json
import numpy as np
import datetime as dt
import urllib
import oauth2
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from sklearn.ensemble import RandomForestClassifier
from nltk.tokenize import RegexpTokenizer
from stop_words import get_stop_words
from nltk.stem.porter import PorterStemmer
#from gensim import corpora, models
#import gensim
from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
import matplotlib.pyplot as plt
from sklearn.cross_validation import KFold
import twitter_search as tw
from NE import name_rec1

#Read channel and video data from JSON files
def read_data():
	V = []
	for i in range(10):
		for j in range(9):
			try:
				with open("video_data/VideoData"+str(i)+"part"+str(j)+".json") as f:
					V = V + json.load(f)
			except:
				break

	with open('ChannelData.json') as cdata:
		Cdata = json.load(cdata)

	return (V, Cdata)

#Clean data
#there are 10 videos withour stat and snippet(i.e. no description, channelId, title...etc), we have to discard theses
#there are 8298 videos without tags (they all have snippet), we can keep them
def clean_data(V):
	stat, comm, snippet, des, pubtime, tags, notag, catId, title, chId = [], [], [], [], [], [], [], [], [], []
	for i in range(len(V)):
		item = V[i]
		v_id = item['videoId']
		if item['statistics'] == None:
			stat.append(v_id)
		if item['comments'] == None:
			comm.append(v_id)
		try:
			if item['snippet']['description'] == None:
				des.append(v_id)
			if item['snippet']['publishedAt'] == None:
				pubtime.append(v_id)
			try:
				if item['snippet']['tags'] == None:
					tags.append(v_id)
			except:
				notag.append(v_id)
			if item['snippet']['categoryId'] == None:
				catId.append(v_id)
			if item['snippet']['title'] == None:
				title.append(v_id)
			if item['snippet']['channelId'] == None:
				chId.append(v_id)
		except:
			snippet.append(i)
	#Discard all the videos without snippet
	for i in range(len(snippet)):
		V.pop(snippet[i]-i)

	return V

#Get the mapping of channel to video
#Get the info of videos
def video_sum(V):
	V_stat, V_comm, V_des, V_pubtime, V_tags, V_catId, V_title, VC_dict = {}, {}, {}, {}, {}, {}, {}, {}
	for item in V:
		v_id = item['videoId']
		V_stat[v_id] = item['statistics']
		V_comm[v_id] = item['comments']
		V_des[v_id] = item['snippet']['description']
		V_pubtime[v_id] = item['snippet']['publishedAt']
		try:
			V_tags[v_id] = item['snippet']['tags']
		except:
			V_tags[v_id] = None
		V_catId[v_id] = item['snippet']['categoryId']
		V_title[v_id] = item['snippet']['title']
		if not VC_dict.has_key(item['snippet']['channelId']):
			VC_dict[item['snippet']['channelId']] = [v_id]
		if VC_dict.has_key(item['snippet']['channelId']):
			VC_dict[item['snippet']['channelId']].append(v_id)

	catId_set = set([V_catId[key] for key in V_catId])

	return (V_stat, V_comm, V_des, V_pubtime, V_tags, V_catId, V_title, VC_dict, catId_set)

#Get info of sampled channels
def channel_sum(Cdata, VC_dict):
	C_tmp = {}
	for item in Cdata:
		c_id = item['channelID']
		if not C_tmp.has_key(c_id):
			C_tmp[c_id] = item
	C, C_stat, C_des, C_title, C_pubtime = {}, {}, {}, {}, {}
	misschannel = []
	for key in VC_dict:
		try:
			channel = C_tmp[key]
			C[key] = channel
			C_stat[key], C_des[key], C_title[key], C_pubtime[key] = channel['statistics'], channel['snippet']['description'], channel['snippet']['title'], channel['snippet']['publishedAt']
		except:
			misschannel.append(key)
			pass
	for channel in misschannel:
		del VC_dict[channel]
	return (C, C_stat, C_des, C_title, C_pubtime)

#Apply KMeans clustering on channel data and return statistics of clustering result, K is number of clusters
def Clustering(C, K):
	# commentCount = [int(item["statistics"][u'commentCount']) for item in C]
	# viewCount = [int(item["statistics"][u'viewCount']) for item in C]
	# videoCount = [int(item["statistics"][u'videoCount']) for item in C]
	# subCount = [int(item["statistics"][u'subscriberCount']) for item in C]
	commentCount = [int(C[item]["statistics"][u'commentCount']) for item in C]
	viewCount = [int(C[item]["statistics"][u'viewCount']) for item in C]
	videoCount = [int(C[item]["statistics"][u'videoCount']) for item in C]
	subCount = [int(C[item]["statistics"][u'subscriberCount']) for item in C]

	mat = np.matrix
	X = mat([commentCount, viewCount, videoCount, subCount]).T

	#KMeans
	random_state = 170
	pred = KMeans(n_clusters=K, random_state=random_state).fit_predict(X)

	#Compute statistics for each cluster
	clusterdata = {str(i):[] for i in range(K)}
	clusterrange = {str(i):0 for i in range(K)}
	clustermean = {str(i):0 for i in range(K)}
	clusterstd = {str(i):0 for i in range(K)}
	clusternumber = {str(i):0 for i in range(K)}

	for i in range(pred.shape[0]):
		for label in range(K):
			if pred[i] == label:
				clusterdata[str(label)].append(X[i, 1])
				break

	for label in range(K):
		clusterrange[str(label)] = (min(clusterdata[str(label)]), max(clusterdata[str(label)]))
		clustermean[str(label)] = np.array(clusterdata[str(label)]).mean()
		clusterstd[str(label)] = np.array(clusterdata[str(label)]).std()
		clusternumber[str(label)] = len(clusterdata[str(label)])

	return (clusterdata, clusterrange, clustermean, clusterstd, clusternumber)

#Applying KMeans clustering assuming cluster number is 3
def Clustering3(C):
	commentCount = [int(item["statistics"][u'commentCount']) for item in C]
	viewCount = [int(item["statistics"][u'viewCount']) for item in C]
	videoCount = [int(item["statistics"][u'videoCount']) for item in C]
	subCount = [int(item["statistics"][u'subscriberCount']) for item in C]

	mat = np.matrix
	X = mat([commentCount, viewCount, videoCount, subCount]).T

	#KMeans
	random_state = 170
	pred = KMeans(n_clusters=3, random_state=random_state).fit_predict(X)

	#Compute statistics for each cluster
	l0, l1, l2, idx0, dx1, idx2 = [], [], [], [], [], []
	for i in range(pred.shape[0]):
		if pred[i] == 0:
			l0.append(X[i,1])
			idx0.append(i)
		if pred[i] == 1:
			l1.append(X[i,1])
			idx1.append(i)
		if pred[i] == 2:
			l2.append(X[i,1])
			idx2.append(i)
	Range0 = (min(l0), max(l0))
	Range1 = (min(l0), max(l0))
	Range2 = (min(l0), max(l0))
	mean0, mean1, mean2 = np.array(l0).mean(), np.array(l1).mean(), np.array(l2).mean()
	std0, std1, std2 = np.array(l0).std(), np.array(l1).std(), np.array(l2).std()
	len0, len1, len2 = len(l0), len(l1), len(l2)
	return (Range0, Range1, Range2, mean0, mean1, mean2, std0, std1, std2, len0, len1, len2)

#Get the PCA representation of quantitation of channel popularity
#Apply on clustering result and plot clusters
def PCA(C_stat, C):
	C_view, C_video, C_sub, C_comm, pred = [], [], [], [], []
	for key in C_stat:
		C_view.append(int(C_stat[key][u'viewCount']))
		C_video.append(int(C_stat[key][u'videoCount']))
		C_sub.append(int(C_stat[key][u'subscriberCount']))
		C_comm.append(int(C_stat[key][u'commentCount']))
		if int(C_stat[key][u'viewCount']) < 850000000:
			pred.append(1)
		elif 850000000 <= int(C_stat[key][u'viewCount']) < 3700000000:
			pred.append(2)
		else:
			pred.append(3)
	#Standardization
	C_view_s, C_video_s, C_sub_s, C_comm_s = [], [], [], []
	viewmean, viewstd = np.array(C_view).mean(), np.array(C_view).std()
	videomean, videostd = np.array(C_video).mean(), np.array(C_video).std()
	submean, substd = np.array(C_sub).mean(), np.array(C_sub).std()
	commmean, commstd = np.array(C_comm).mean(), np.array(C_comm).std()
	for i in range(len(C_view)):
		C_view_s.append((C_view[i]-viewmean)/viewstd)
		C_video_s.append((C_video[i]-videomean)/videostd)
		C_sub_s.append((C_sub[i]-submean)/substd)
		C_comm_s.append((C_comm[i]-commmean)/commstd)

	Pmatrix = np.matrix([C_view_s, C_video_s, C_sub_s, C_comm_s]).T
	pca = PCA(n_components=2)
	pca.fit(Pmatrix)
	Xpca = pca.transform(Pmatrix)

	plt.scatter(Xpca[:, 0], Xpca[:, 1], c = pred)
	plt.savefig('pcachannelstandard.jpg')

	i = 0
	pcadict = {}
	for key in C:
		pcadict[key] =  np.ndarray.tolist(Xpca[i,:])
		i = i + 1

	with open("Xpca.txt", "a") as f:
		f.write(str(pcadict))
	return Xpca

#Generate part of channel features including (as matrix mat):
	#Duration, video publishing frequency
	#Ratio of subscriber/view, comment/view
	#Sentiment anaylysis result: positive/negative/neutral/compound sentiment scores
#Generate response vector y according to clustering result:
	#cluster1: [0, 850000000], cluster2:[850000000, 3700000000], cluster3[3700000000, infinity]
def Cgen(C, C_stat, C_des, C_title, C_pubtime, V_stat, V_comm, VC_dict):
	T, v_freq, sub_view, comm_view, comm_sentiment_Neg, comm_sentiment_Neu, comm_sentiment_Pos, comm_sentiment_Comp = [], [], [], [], [], [], [], []
	y = []
	keys = C.keys()
	end = dt.datetime.now()
	for key in C:
		#Get the time of each channel
		t = str(C_pubtime[key])
		start = dt.datetime(year = int(t[:4]), month = int(t[5:7]), day = int(t[8:10]), hour = int(t[11:13]), minute = int(t[14:16]), second = int(t[17:19]), microsecond = int(t[20:23]))
		T.append((end - start).days)
		#Get the video publishing frequency of each channel
		v_freq.append(float(C_stat[key]['videoCount'])/float((end - start).days))
		#Get the subscriber/viewer and comment/viewer ratio
		if float(C_stat[key]['subscriberCount']) == 0 or float(C_stat[key]['viewCount']) == 0 or float(C_stat[key]['commentCount']) == 0:
			sub_view.append(0)
			comm_view.append(0)
		else:
			sub_view.append(float(C_stat[key]['subscriberCount'])/float(C_stat[key]['viewCount']))
			comm_view.append(float(C_stat[key]['commentCount'])/float(C_stat[key]['viewCount']))
		#Process the comments of videos of each channel
		sid = SentimentIntensityAnalyzer()
		cs = np.matrix([0.0,0.0,0.0,0.0])
		for video in VC_dict[key]:
			if V_comm[video] != []:
				vs = np.matrix([0.0,0.0,0.0,0.0])
				for sentence in V_comm[video]:
					score = sid.polarity_scores(sentence)
					vs[0,0] = vs[0,0] + score['neg']
					vs[0,1] = vs[0,1] + score['neu']
					vs[0,2] = vs[0,2] + score['pos']
					vs[0,3] = vs[0,3] + score['compound']
				cs = cs + float(V_stat[video][u'viewCount']) * vs/float(len(V_comm[video]))
		cs = cs/float(len(VC_dict[key]))
		comm_sentiment_Neg.append(cs[0,0])
		comm_sentiment_Neu.append(cs[0,1])
		comm_sentiment_Pos.append(cs[0,2])
		comm_sentiment_Comp.append(cs[0,3])
		#Get response as categorical labels
		if float(C_stat[key]['viewCount']) < 850000000:
			y.append('1')
		elif 850000000 <= float(C_stat[key]['viewCount']) < 3700000000:
			y.append('2')
		else:
			y.append('3')
	mat = np.matrix([T, v_freq, sub_view, comm_view, comm_sentiment_Neg, comm_sentiment_Neu, comm_sentiment_Pos, comm_sentiment_Comp])
	return (np.array(y), mat)

#Aggregate the video statistics for each channel including:
	#Ratio of like/dislike, comment/view, favorite/view
def Vgen(V_stat, VC_dict):
	like_dislike, comm_view, fav_view = [], [], []
	for video in V_stat:
		if not V_stat[video].has_key(u'dislikeCount'):
			V_stat[video][u'dislikeCount'] = u'0'
		if not V_stat[video].has_key(u'likeCount'):
			V_stat[video][u'likeCount'] = u'0'
		if not V_stat[video].has_key(u'favouriteCount'):
			V_stat[video][u'favouriteCount'] = u'0'
		if V_stat[video][u'viewCount'] == u'0':
			V_stat[video][u'viewCount'] = u'1000000000000000000000000000'
	for key in VC_dict:
		vlist = VC_dict[key]
		tmp = 0
		for video in vlist:
			if int(V_stat[video]['dislikeCount']) == 0:
				tmp = tmp + float(V_stat[video]['likeCount'])
			else:
				tmp = tmp + float(V_stat[video]['likeCount'])/float(V_stat[video]['dislikeCount'])
		like_dislike.append(tmp/float(len(vlist)))
		#slike_dislike.append(sum([float(V_stat[item]['likeCount'])/float(V_stat[item]['dislikeCount']) for item in vlist])/float(len(vlist)))
		comm_view.append(sum([float(float(V_stat[item]['commentCount'])/float(V_stat[item]['viewCount'])) for item in vlist])/float(len(vlist)))
		fav_view.append(sum([float(float(V_stat[item]['favouriteCount'])/float(V_stat[item]['viewCount'])) for item in vlist])/float(len(vlist)))
	V_agg = np.matrix([like_dislike, comm_view, fav_view])
	return V_agg

#Process and aggregate the category info of videos of a channel as dummy variable (44 categorys, 43 dummy variables)
def cat_dummy(VC_dict, V_catId, V_stat):
	cat_id_dummy = []
	for channel in VC_dict:
		video_cat = [0 for i in range(43)]
		for video in VC_dict[channel]:
			if int(V_catId[video])!=44:
				video_cat[int(V_catId[video])-1] = video_cat[int(V_catId[video])-1] + 1
		cat_id_dummy.append([item/float(len(VC_dict[channel])) for item in video_cat])
	cat_id_dummy = np.matrix(cat_id_dummy).T
	return cat_id_dummy

#Compute the referrer scores for each channel: sum(sum of each video's referrer's followers/1000.0)
def ref_scores():
	tweets = tw.combine()
	Vref = tw.check2(tweets)
	Vref1 = {channel:0 for channel in VC_dict}
	for channel in VC_dict:
		for key in VC_dict[channel]:
			try:
				if key[0] == '-':
					key = key[1:]
				if Vref[key] != {}:
					Vref1[channel] = Vref1[channel] + sum([Vref[key][k]/1000.0 for k in Vref[key]])
			except:
				pass
	Ref_scores = np.matrix([Vref1[key] for key in Vref1])
	return Ref_scores

#Identify if channel has twitter, facebook, instagram google+
def socialmedia(C):
	score = {channel:0 for channel in C}
	for channel in C:
		if C[channel]['snippet']['description'].find('twitter')!=-1 or C[channel]['snippet']['description'].find('Twitter')!=-1:
			score[channel] = score[channel] + 1
		if C[channel]['snippet']['description'].find('facebook')!=-1 or C[channel]['snippet']['description'].find('Facebook')!=-1:
			score[channel] = score[channel] + 1
		if C[channel]['snippet']['description'].find('instagram')!=-1 or C[channel]['snippet']['description'].find('Instagram')!=-1:
			score[channel] = score[channel] + 1
		if C[channel]['snippet']['description'].find('google')!=-1 or C[channel]['snippet']['description'].find('Google')!=-1:
			score[channel] = score[channel] + 1
	social = np.matrix([score[channel] for channel in C])
	return social

#get the channels with twitter accounts:
def get_account(C):
	accounts = {}
	for channel in C:
		tmp = C[channel]['snippet']['description'].find('twitter')
		if tmp != -1:
			accounts[channel] = C[channel]['snippet']['description'][tmp:tmp+50]
	return accounts

#Named Entity Recognition: compute the average occuring times of named entities in description and titles for each video of each channel
def C_names(VC_dict, V_des, V_title):
    C_name_map = {}
    C_name = []
    for channel in VC_dict:
        videos = VC_dict[channel]
        num = 0
        for video in videos:
            num = num + len(name_rec1(V_des[video])) + len(name_rec1(V_title[video]))
        ave = num/float(len(videos))
        C_name_map[channel] = ave
        C_name.append(ave)
    C_name = np.matrix(C_name)
    return (C_name_map, C_name)

#Aggregate all the features:
	#C_name is score of named entity recognition
	#cat_id_dummy are ecoded dummy features of video categories
	#Ref_scores are aggregated referrer scores
	#Social media scores
def Agg1(mat, V_agg, C_name, Ref_scores, cat_id_dummy, social, y, C):
	X = np.concatenate((mat, V_agg, C_name, Ref_scores, cat_id_dummy, social))
	i = 0
	outdict, ydict = {}, {}
	for key in C:
		outdict[key] = np.ndarray.tolist(X[i,:])[0]
		ydict[key] = y[i]
		i = i + 1
	with open("features.txt", "a") as f:
		f.write(str(outdict))
	with open("response.txt", "a") as ff:
		ff.write(str(ydict))
	return (X.T, y)

#Aggregate all the processors
def PROCESSER():
	(V, Cdata) = read_data()
	V = clean_data(V)
	(V_stat, V_comm, V_des, V_pubtime, V_tags, V_catId, V_title, VC_dict, catId_set) = video_sum(V)
	(C, C_stat, C_des, C_title, C_pubtime) = channel_sum(Cdata)
	(y, mat) = Cgen(C, C_stat, C_des, C_title, C_pubtime, V_stat, V_comm, VC_dict)
	V_agg = Vgen(V_stat, VC_dict)
	cat_id_dummy = cat_dummy(VC_dict, V_catId, V_stat)
	Ref_scores = ref_scores()
	social = socialmedia(C)
	(C_name_map, C_name) = C_names(VC_dict, V_des, V_title)
	(X, y) = Agg1(mat, V_agg, C_name, Ref_scores, cat_id_dummy, social, y)
	return (X, y)


import cv2
import numpy as np
import matplotlib.pyplot as plt

img = cv2.imread('./images/image.jpg', cv2.IMREAD_GRAYSCALE)
ret, img = cv2.threshold(img, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)

#  cross kernel
kernel = cv2.getStructuringElement(cv2.MORPH_CROSS, (5, 5))

# dilation
dilated = cv2.dilate(img, kernel, iterations=3)

# ploting
fig, axs = plt.subplots(1, 2, figsize=(10, 5))
axs[0].imshow(img, cmap='gray')
axs[0].set_title('Image')
axs[1].imshow(dilated, cmap='gray')
axs[1].set_title('Dilation')
plt.show()


import qrcode

na_chto = input('На что куракод ')

version = int(input('размер поля '))
box_size = int(input('сложность куракода, не больше 3 '))

qr = qrcode.QRCode(version, box_size, border = 5)

qr.add_data(na_chto)

qr.make(fit=True)
img = qr.make_image(fill_color='black', back_color='white')

name = input('Назови меня ')

img.save(f'{name}.png')

print('Куракод создан')


from setuptools import find_packages, setup

def readfile(filename):
    with open(filename, "r+") as f:
        return f.read()

setup(
    name="medusa",
    version="1.1.0",
    description="Photo editor helpful for ML image preprocessing.",
    long_description=readfile("README.md"),
    url="https://github.com/sqoshi/medusa",
    author="Piotr Popis",
    author_email="piotrpopis@icloud.com",
    license="MIT",
    py_modules=["run"],
    packages=find_packages(),
    zip_safe=False,
    entry_points={
        "console_scripts": [
            "medusa = run:main",
        ]
    },
    install_requires=[
        "Pillow==8.2.0",
        "termcolor==1.1.0",
        "mtcnn==0.1.0"
    ],
)


import pytest
from mixer.backend.django import mixer
from .fixture import (full_text, wanted)

pytestmark = pytest.mark.django_db

class TestPost:
    def test_init(self):
        obj = mixer.blend('posts.BlogPost')
        assert obj.pk == 1, 'Should save an instance'

    def test_get_excerpt(self):
        obj = mixer.blend('posts.BlogPost', text=full_text)
        result = obj.get_excerpt()
        expected = wanted
        assert result == expected, 'should be 3 lines'

    def test_author(self):
        obj = mixer.blend('posts.BlogPost', author=None)

        result = obj.author
        expected = 'Anon'
        assert result == expected, 'Should be Anon'


from fastapi import FastAPI
from fastapi import FastAPI, Request
from newpaddle import OCR, parsing
from paddleocr import PaddleOCR
import cv2
import base64
import skimage.io
from uuid import uuid4
import os
def decode(base64_string):
    if isinstance(base64_string, bytes):
        base64_string = base64_string.decode("utf-8")

    imgdata = base64.b64decode(base64_string)
    img = skimage.io.imread(imgdata, plugin='imageio')
    return img

def remove_alpha_channel(img):
    if img.shape[2] == 4:
        img = img[:, :, :3]
    return img

app = FastAPI()
ocr = PaddleOCR(use_angle_cls=True, lang='en',show_log = False)

if not os.path.exists("image_upload"):
    os.mkdir("image_upload")

@app.get("/")
async def root():
    return {"message": "Server is up and running!"}

@app.post("/scan")
async def scan_image(request:Request):

    try:

        json_input = await request.json()
        image_value = json_input['image']
        image_base64 = image_value.split(',')[1]
        image=decode(image_base64)

        image = remove_alpha_channel(image)
        cv2.imwrite(f"image_upload/{uuid4().hex}.jpg", image)
        list_of_words = OCR(image, ocr)
        final_results = parsing(list_of_words)
        return final_results

    except Exception as e:
        print(e)
        # empty_directory("image_upload")
        return {"error": str(e)}


""" Util functions to split data into train/val. """
from __future__ import print_function, division

import os
from pathlib import Path

import numpy as np
import pandas as pd

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

from pandas.api.types import is_string_dtype
from collections import OrderedDict
from sklearn.preprocessing import LabelEncoder

from sklearn.model_selection import ShuffleSplit, KFold
from sklearn.model_selection import GroupShuffleSplit, GroupKFold
from sklearn.model_selection import StratifiedShuffleSplit, StratifiedKFold

def cv_splitter(cv_method: str='simple', cv_folds: int=1, test_size: float=0.2,
                mltype: str='reg', shuffle: bool=False, random_state=None):
    """ Creates a cross-validation splitter.
    Args:
        cv_method: 'simple', 'stratify' (only for classification), 'groups' (only for regression)
        cv_folds: number of cv folds
        test_size: fraction of test set size (used only if cv_folds=1)
        mltype: 'reg', 'cls'
    """
    # Classification
    if mltype == 'cls':
        if cv_method == 'simple':
            if cv_folds == 1:
                cv = ShuffleSplit(n_splits=cv_folds, test_size=test_size, random_state=random_state)
            else:
                cv = KFold(n_splits=cv_folds, shuffle=shuffle, random_state=random_state)

        elif cv_method == 'stratify':
            if cv_folds == 1:
                cv = StratifiedShuffleSplit(n_splits=cv_folds, test_size=test_size, random_state=random_state)
            else:
                cv = StratifiedKFold(n_splits=cv_folds, shuffle=shuffle, random_state=random_state)

    # Regression
    elif mltype == 'reg':
        # Regression
        if cv_method == 'group':
            if cv_folds == 1:
                raise ValueError('GroupShuffleSplit splits groups based on group count and not based on sample count.')
                # https://github.com/scikit-learn/scikit-learn/issues/13369
                # https://github.com/scikit-learn/scikit-learn/issues/9193
                cv = GroupShuffleSplit(n_splits=cv_folds, test_size=test_size, random_state=random_state)
            else:
                cv = GroupKFold(n_splits=cv_folds)

        elif cv_method == 'simple':
            if cv_folds == 1:
                cv = ShuffleSplit(n_splits=cv_folds, test_size=test_size, random_state=random_state)
            else:
                cv = KFold(n_splits=cv_folds, shuffle=shuffle, random_state=random_state)
    return cv

def plot_ytr_yvl_dist(ytr, yvl, title=None, outpath='.'):
    """ Plot distributions of response data of train and val sets. """
    fig, ax = plt.subplots()
    plt.hist(ytr, bins=100, label='ytr', color='b', alpha=0.5)
    plt.hist(yvl, bins=100, label='yvl', color='r', alpha=0.5)
    if title is None: title = ''
    plt.title(title)
    plt.tight_layout()
    plt.grid(True)
    plt.legend()
    if outpath is None:
        plt.savefig(Path(outpath)/'ytr_yvl_dist.png', bbox_inches='tight')
    else:
        plt.savefig(outpath, bbox_inches='tight')


import unittest
import cProfile, pstats

import swimpy
import tests
from mpi4py import MPI

COMM = MPI.COMM_WORLD
RANK, SIZE = COMM.Get_rank(), COMM.Get_size()

class TestMpi(tests.ProjectTestCase):
    @classmethod
    def setUpClass(self):
        if RANK == 0:
            super(TestMpi, self).setUpClass()
            self.project.config_parameters(nbyr=2)
        COMM.Barrier()
        self.project = swimpy.Project(tests.SWIM_TEST_PROJECT)

    @classmethod
    def tearDownClass(self):
        self.project.browser.settings.unset()
        COMM.Barrier()
        if RANK == 0:
            super(TestMpi, self).tearDownClass()

    def test_run_parallel(self):
        n = int(SIZE*1.5)
        args = [dict(smrate=i/10.) for i in range(n)]
        runs = self.project.cluster.run_parallel(
            args=args, parallelism='mpi')
        self.assertEqual(runs.count(), n)
        cloneids = set([int(t.split()[1].split('_')[-1])
                        for t in sorted(runs.values_list('tags', flat=True))])
        self.assertListEqual(list(cloneids), list(range(min(SIZE, n))))

if __name__ == '__main__':
    cProfile.run('unittest.main()', 'pstats')
    if RANK == 0:
        # print profile stats ordered by time
        pstats.Stats('pstats').strip_dirs().sort_stats('time').print_stats(5)


import csv
import sqlite3

from flask import Flask, request, g
from collections import Counter

app = Flask(__name__)
app.config.from_object(__name__)

app.config.update(dict(
    DATABASE='/var/www/html/flaskapp/natlpark.db'
))

def connect_to_database():
    return sqlite3.connect(app.config['DATABASE'])

def get_db():
    db = getattr(g, 'db', None)
    if db is None:
        db = g.db = connect_to_database()
    return db

@app.teardown_appcontext
def close_connection(exception):
    db = getattr(g, 'db', None)
    if db is not None:
        db.close()

def execute_query(query, args=()):
    cur = get_db().execute(query, args)
    rows = cur.fetchall()
    cur.close()
    return rows

@app.route('/')
def hello_world():
  return 'Hello from Flask!'

@app.route('/countme/<input_str>')
def count_me(input_str):
    input_counter = Counter(input_str)
    response = []
    for letter, count in input_counter.most_common():
        response.append('"{}": {}'.format(letter, count))
    return '
'.join(response)

@app.route("/viewdb")
def viewdb():
    rows = execute_query("""SELECT * FROM natlpark""")
    return '
'.join(str(row) for row in rows)

@app.route("/state/<state>")
def sortby(state):
    rows = execute_query("""SELECT * FROM natlpark WHERE state = ?""",
                         [state.title()])
    return '
'.join(str(row) for row in rows)

if __name__ == '__main__':
  app.run()


from bs4 import BeautifulSoup
import requests
import json
import re
import pprint

pp = pprint.PrettyPrinter(indent=4)

def getMonstersUrlGenerator():
    r = requests.get('https://roll20.net/compendium/dnd5e/Rules:Monsters%20by%20Name#content')
    data = r.text
    soup = BeautifulSoup(data, 'html.parser')
    urls = []
    monsterUrls = soup.find_all('a', { 'href': True })
    for url in monsterUrls:
        monsterUrl = url['href']
        pp.pprint(monsterUrl)
        if 'compendium/dnd5e/Monsters' in monsterUrl:
            yield 'https://roll20.net' + monsterUrl

def discoverAttributes():
    attributes = {}
    for url in getMonstersUrlGenerator():
        pp.pprint('Doing {0}'.format(url))
        r = requests.get(url)
        data = r.text
        soup = BeautifulSoup(data, 'html.parser')
        for attrItem in soup.find_all(class_='attrListItem'):
            # Save and SavingThrows are the same
            attrName = attrItem.find_next(class_='attrName').text.replace(' ', '')
            attrValue = attrItem.find_next(class_='value').text.strip()
            if not attrName in attributes:
                attributes[attrName] = attrValue

    return attributes

if __name__ == '__main__':
    attr = discoverAttributes()
    pp.pprint(attr)


import cv2
import itertools, os, time
import numpy as np
from Model import get_Model
from parameter import letters
import argparse
from keras import backend as K
K.set_learning_phase(0)

Region = {"A": "서울 ", "B": "경기 ", "C": "인천 ", "D": "강원 ", "E": "충남 ", "F": "대전 ",
          "G": "충북 ", "H": "부산 ", "I": "울산 ", "J": "대구 ", "K": "경북 ", "L": "경남 ",
          "M": "전남 ", "N": "광주 ", "O": "전북 ", "P": "제주 "}
Hangul = {"dk": "아", "dj": "어", "dh": "오", "dn": "우", "qk": "바", "qj": "버", "qh": "보", "qn": "부",
          "ek": "다", "ej": "더", "eh": "도", "en": "두", "rk": "가", "rj": "거", "rh": "고", "rn": "구",
          "wk": "자", "wj": "저", "wh": "조", "wn": "주", "ak": "마", "aj": "머", "ah": "모", "an": "무",
          "sk": "나", "sj": "너", "sh": "노", "sn": "누", "fk": "라", "fj": "러", "fh": "로", "fn": "루",
          "tk": "사", "tj": "서", "th": "소", "tn": "수", "gj": "허"}

def decode_label(out):
    # out : (1, 32, 42)
    out_best = list(np.argmax(out[0, 2:], axis=1))  # get max index -> len = 32
    out_best = [k for k, g in itertools.groupby(out_best)]  # remove overlap value
    outstr = ''
    for i in out_best:
        if i < len(letters):
            outstr += letters[i]
    return outstr

def label_to_hangul(label):  # eng -> hangul
    region = label[0]
    two_num = label[1:3]
    hangul = label[3:5]
    four_num = label[5:]

    try:
        region = Region[region] if region != 'Z' else ''
    except:
        pass
    try:
        hangul = Hangul[hangul]
    except:
        pass
    return region + two_num + hangul + four_num

parser = argparse.ArgumentParser()
parser.add_argument("-w", "--weight", help="weight file directory",
                    type=str, default="Final_weight.hdf5")
parser.add_argument("-t", "--test_img", help="Test image directory",
                    type=str, default="./DB/test/")
args = parser.parse_args()

# Get CRNN model
model = get_Model(training=False)

try:
    model.load_weights(args.weight)
    print("...Previous weight data...")
except:
    raise Exception("No weight file!")

test_dir =args.test_img
test_imgs = os.listdir(args.test_img)
total = 0
acc = 0
letter_total = 0
letter_acc = 0
start = time.time()
for test_img in test_imgs:
    img = cv2.imread(test_dir + test_img, cv2.IMREAD_GRAYSCALE)

    img_pred = img.astype(np.float32)
    img_pred = cv2.resize(img_pred, (128, 64))
    img_pred = (img_pred / 255.0) * 2.0 - 1.0
    img_pred = img_pred.T
    img_pred = np.expand_dims(img_pred, axis=-1)
    img_pred = np.expand_dims(img_pred, axis=0)

    net_out_value = model.predict(img_pred)

    pred_texts = decode_label(net_out_value)

    for i in range(min(len(pred_texts), len(test_img[0:-4]))):
        if pred_texts[i] == test_img[i]:
            letter_acc += 1
    letter_total += max(len(pred_texts), len(test_img[0:-4]))

    if pred_texts == test_img[0:-4]:
        acc += 1
    total += 1
    print('Predicted: %s  /  True: %s' % (label_to_hangul(pred_texts), label_to_hangul(test_img[0:-4])))

    # cv2.rectangle(img, (0,0), (150, 30), (0,0,0), -1)
    # cv2.putText(img, pred_texts, (5, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255,255,255),2)

    #cv2.imshow("q", img)
    #if cv2.waitKey(0) == 27:
    #   break
    #cv2.destroyAllWindows()

end = time.time()
total_time = (end - start)
print("Time : ",total_time / total)
print("ACC : ", acc / total)
print("letter ACC : ", letter_acc / letter_total)


#!/usr/bin/env python
# coding: utf-8

# ## eaze-ml   ¯\\_(ツ)_/¯
# #### Data Science FrameWork

# In[11]:

## Basic Libraries
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import statsmodels.api as sm
from statsmodels.stats.outliers_influence import variance_inflation_factor
## interactive Visualization
import plotly.graph_objs as go
import plotly.express as px
import plotly.figure_factory as ff
from plotly.subplots import make_subplots
import cufflinks
cufflinks.go_offline(connected=True)
from plotly.offline import iplot# init_notebook_mode(connected=True)
import plotly.offline as pyo
pyo.init_notebook_mode()
#### Ignore Warning Messages
import os
import warnings
warnings.filterwarnings('ignore')

#### for Preprocessing
from sklearn.preprocessing import LabelEncoder,OneHotEncoder,LabelBinarizer
from sklearn.preprocessing import MinMaxScaler,StandardScaler
from sklearn.pipeline import make_pipeline

#### Machine Learning Essentials
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score,GridSearchCV

#### Evaluation Metrices
from sklearn.utils.multiclass import unique_labels
from sklearn import metrics
from sklearn.metrics import classification_report, confusion_matrix, f1_score
from sklearn.metrics import balanced_accuracy_score,roc_auc_score,log_loss,accuracy_score
from sklearn.metrics import mean_squared_error,r2_score,mean_squared_log_error

#### Machine Learning Models
### supervised
from sklearn.linear_model import LogisticRegression,LinearRegression
from sklearn.ensemble import RandomForestClassifier,RandomForestRegressor
from sklearn.ensemble import GradientBoostingClassifier,GradientBoostingRegressor
from sklearn.ensemble import AdaBoostClassifier,AdaBoostRegressor
from xgboost import XGBClassifier,XGBRegressor
from lightgbm import LGBMClassifier,LGBMRegressor
import lightgbm as lgb
from sklearn.svm import SVR,SVC
from sklearn.neighbors import KNeighborsClassifier,KNeighborsRegressor
### unsupervised
from sklearn.cluster import KMeans
### Neural Network
from sklearn.neural_network import MLPClassifier
from sklearn.naive_bayes import GaussianNB

#### Feature Selection
from sklearn.feature_selection import RFE
from sklearn.decomposition import PCA

# In[10]:

### import data
def importdata(filename):
    '''
    Import Data from csv or excel in dataframe format
    example usage:
        dataframe = importtain('filename')
        just specify filename
    '''
    x = filename.split('.')[1]
    if x == 'xlsx':
        dataframe = pd.read_excel(filename)
    else:
        dataframe = pd.read_csv(filename)

    print('Dataframe Imported Successfully')
    return dataframe

def info(dataframe):
    '''
    ## Gives Entire Summary info of dataframe like nulls,uniques,
    percentage missing value,type of column of dataframe.
    ## gives you entire details of dataFrame
    '''
    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
    num = dataframe.select_dtypes(include=numerics).columns
    cat = dataframe.drop(num,axis=1)
    cat = cat.columns

    #cols = dataframe.columns
    #num = dataframe._get_numeric_data().columns
    #cat = list(set(cols) - set(num_cols))
    print('*** Dataset Infomarion ***')
    print('#####'*10)
    print ("No of Rows     : " ,dataframe.shape[0])
    print ("No of columns  : " ,dataframe.shape[1])
    print ('No of Numerical columns:',len(num))
    print ('No of Categorical columns:',len(cat))
    print('#####'*10)
    print ("\n Total Missing values :  ",dataframe.isnull().sum().values.sum(),'\n')
    print('#####'*10)
    types = []
    for i in dataframe.columns:
        if i in num:
            types.append('Numerical')
        else:
            types.append('Categorical')
    temp = pd.DataFrame()
    temp['Column Name'] = dataframe.columns
    temp['Nulls/NaN']= dataframe.isnull().sum().values
    temp['outof'] = dataframe.shape[0]
    temp['Unique'] = dataframe.nunique().values
    temp['Type of Columns'] = types
    print("Summary of DataFrame:\n\n",temp)
    if dataframe.isnull().sum().sum() > 0:
        missingdata(dataframe)

def missingdata(data):
    '''
    Plots Percentage missing value in barplot format
    '''
    total = data.isnull().sum().sort_values(ascending = False)
    percent = (data.isnull().sum()/data.isnull().count()*100).sort_values(ascending = False)
    ms=pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
    ms= ms[ms["Percent"] > 0]
    f,ax =plt.subplots(figsize=(8,6))
    plt.xticks(rotation='90')
    fig=sns.barplot(ms.index, ms["Percent"],color="green",alpha=0.8)
    plt.xlabel('Features', fontsize=15)
    plt.ylabel('Percent of missing values', fontsize=15)
    plt.title('Percent missing data by feature', fontsize=15)
    plt.show()

def getcolumnstype(dataframe):
    '''
   Returns categorical and numerical columns names from dataframe
   example usage:
           num, cat = getcolumnstype(dataframe)
    '''
    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
    num = dataframe.select_dtypes(include=numerics).columns
    cat = dataframe.drop(num,axis=1)
    cat = cat.columns
    return num,cat

def detail_cat(dataframe):
    '''
    Prints detial of categoy columns
    '''
    num, cat = getcolumnstype(dataframe)
    print('Deatils of Categorical Columns:\n\n')
    for i in cat:
        print(i,">> column has values :\n",dataframe[i].value_counts())
        print("* * "*10)

## for dropping columns just name as maany as columns
def dropcolumns(dataframe,*argv):
    '''
    Drops columns specified from dataframe and return dataframe
    example usage:
        dataframe = dropcolumns(dataframe,'columnam')
        specify column names in '' seprated by ','comma
    '''
    columns = list(argv)
    dataframe.drop(columns,axis=1,inplace=True)
    return dataframe
# -------- advance --------------
def drop_columns(dataframe,n=0,*argv):
    '''
    Drop columns which have null value percentage above specified in n
    example usage:
            dataframe = drop_columns(dataframe,n,column names)
            n = set to 0 if don't wan't to use percenatage missing feature
            example drop_columns(df,0,'id')
    '''
    if n > 0:
        data = dataframe.copy()
        total = data.isnull().sum().sort_values(ascending = False)
        percent = (data.isnull().sum()/data.isnull().count()*100).sort_values(ascending = False)
        ms=pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
        ms= ms[ms["Percent"] > 0]
        print("Null value summary: \n",ms)
        dropout = percent[percent > n].index
        print('columns dropped',dropout)
        data.drop(dropout,axis=1,inplace=True)

        columns = list(argv)
        data.drop(columns,axis=1,inplace=True)
        print('\n\ncolumns dropped \n\n',list(dropout),"\n",columns)
        return data
    else:
        columns = list(argv)
        dataframe.drop(columns,axis=1,inplace=True)
        print("\n\n",columns,"> Dropped \n\n")
        return dataframe

def get_IQR(dataframe):
    """
    Prints IQR table for dataframe
    """
    Q1 = dataframe.quantile(0.25)
    Q3 = dataframe.quantile(0.75)
    IQR = Q3-Q1
    print(IQR)

def show_outlier(df):
    '''
    show no of outliers in each column
    '''
    for i in df._get_numeric_data().columns:
        low,high = df[i].quantile([0.1,0.95]).values
        rows = df[(df[i] > low) & (df[i] < high)].shape[0]
        if rows > 0:
            print(rows,'  No of outliers in column >>>',i)
        else:
            print('***** no outlier in ',i)

def remove_outlier(dataframe):
    '''
    Removes all outlier data from dataframe from entire dataframe
    example usage:
        newdf = remove_outlier(dataframe)
        # newdf will have no outlier values
    '''
    Q1 = dataframe.quantile(0.5)
    Q3 = dataframe.quantile(0.95)
    IQR = Q3-Q1
    newdataframe = dataframe[~((dataframe < (Q1 - 1.5 * IQR)) |(dataframe > (Q3 + 1.5 * IQR))).any(axis=1)]
    print('IQR Range for all columns\n',IQR)
    return newdataframe

def fillnulls(dataframe,value,*argv):
    '''
    Fill null values in dataframe columns with specified value or with mean,median,mode
    example usage:
            train = fillnulls(train,'median','batch_enrolled'...)
            fills median value of respctive column in place of null values

        if wan't to specify any other values beside mean,median,mode
        just specigy that value:
            train = fillnulls(train,'value','batch_enrolled'...)
    '''
    dataframe1 = dataframe.copy()
    columns = list(argv)
    print('\nColumns to be cleaned are :\n',columns,'\n')
    if value == 'mean':
        for i in columns:
            val = dataframe1[i].mean()
            dataframe1[i] = dataframe1[i].fillna(val)
            print(i,'filled with:',val,'\n')
        return dataframe1
    elif value == 'median':
        for i in columns:
            val = dataframe1[i].median()
            dataframe1[i] = dataframe1[i].fillna(val)
            print(i,'filled with:',val,'\n')
        return dataframe1
    elif value == 'mode':
        for i in columns:
            val = dataframe1[i].mode
            dataframe1[i] = dataframe1[i].fillna(val)
            print(i,'filled with:',val,'\n')
        return dataframe1
    else:
        for i in columns:
            dataframe1[i] = dataframe1[i].fillna(value)
            print("\n",value,'Filled inplace of NaNs\n')
        return dataframe1

def extract_number(dataframe,columname):
    '''
    extract numbers from column
    example usage:
            dataframe = extract_number(dataframe,'columname')
        just spcify dataframe and columname from which no to be extracted
        and it return cleaned column
        Null values will be replaced with 9999
    '''
    if dataframe[columname].isnull().sum() > 0:
        dataframe[columname].fillna('9999',inplace=True)

    dataframe[columname] = dataframe[columname].astype(str)
    dataframe[columname].replace("[^0-9]","",regex=True,inplace=True)
    dataframe[columname] = dataframe[columname].apply(lambda x: x.strip())
    return dataframe

def label_encode(dataframe):
    '''
    Perfoms label Encoding on all categorical features and return one label encoded data
    usage example :
            dataframe = onehot_encode(dataframe)
    '''
    data = dataframe.copy()
    num,cat = getcolumnstype(data)
    print('\ncolumns label encoded\n',cat)
    data[cat] = data[cat].apply(LabelEncoder().fit_transform)
    return data

def onehot_encode(dataframe):
    '''
    Perfoms One Hot Encoding on all categorical features and return one hot encoded data
    usage example :
            dataframe = onehot_encode(dataframe)
    '''
    data = dataframe.copy()
    num,cat = getcolumnstype(data)
    print('\ncolumns onehot encoded\n',cat)
    #data[cat] = data[cat].apply(OneHotEncoder().fit_transform)
    data = pd.get_dummies(data,drop_first=True)
    return data

def rfe(model,X_train,y_train,n=10):
    '''
    Performs Recurssive Feature Elimination and return best n feaures specified (defualt set to 10)
    example usage:
            rfecolumns = rfe(model,X_train,y_train,n)
        model - machine learning algorithm model
        X_train,y_train - generated from train_test_split
        n - (optional) set no of features to be selected
    '''
    rfe = RFE(model, n)
    print('\nExecuting RFE\nPlease wait.........')
    X_train = scale(X_train)
    rfe.fit(X_train,y_train)
    print('### RFE selected columns:\n',X_train.columns[rfe.support_])
    return X_train.columns[rfe.support_]

def scale(dataframe):
    mm = MinMaxScaler()
    c = mm.fit_transform(dataframe)
    df = pd.DataFrame(c,columns = dataframe.columns)
    return df

def VIF(dataframe):
    '''
    Prints Variance Inflation Factor value for each column in dataframe in tabluar format
    '''
    vif = pd.DataFrame()
    vif['feature'] = dataframe.columns
    vif['vif'] = [variance_inflation_factor(dataframe.values,i)for i in range(dataframe.shape[1])]
    vif['vif'] = round(vif['vif'],2)
    vif = vif.sort_values(by='vif',ascending=False)
    return print('\n\n******** VIF *********\n\n',vif)

def stat_models(y, X):
    '''
    Display statastical table for X and Y
    '''
    model = sm.OLS(y, X)
    results = model.fit()
    print("***"*20)
    print("\n\n",results.summary())

def multiclass_roc_auc_score(y_test, y_pred, average="macro"):
    lb = LabelBinarizer()
    lb.fit(y_test)
    y_test = lb.transform(y_test)
    y_pred = lb.transform(y_pred)
    return roc_auc_score(y_test, y_pred, average=average)

def classification_result(y_test,y_pred):
    '''
    Evaluate Performace with Popular Metrices:
    gives different scores when passed y_true & y_pred
    example usage:
            regression_result(y_true,y_pred)

    '''
    print("--------------------------------------------------------")
    print('********************* Model Results ********************')
    print("--------------------------------------------------------")
    print("F1 Score :",f1_score(y_test,y_pred,average = "weighted"))
    print('AUC-ROC Score :',multiclass_roc_auc_score(y_test, y_pred)) ## only for Binary Classification
    print('Report:\n',classification_report(y_test, y_pred))
    confusion_mat(y_test,y_pred)
    if len(y_test.value_counts()) == 2:
        roc_curve_graph(y_test, y_pred)
    print("--------------------------------------------------------")

def regression_result(y_test,y_pred):
    '''
    Evaluate Performace with Popular Metrices:
    gives different scores when passed y_true & y_pred
    example usage:
            regression_result(y_true,y_pred)

    '''
    print("-----------------------------------------")
    print('************* Model Results *************')
    print("-----------------------------------------")
    print('R2 score:',r2_score(y_test,y_pred))
    print('MSE score:',mean_squared_error(y_test, y_pred))
    rmse = np.sqrt(mean_squared_error(y_test, y_pred))
    print('RMSE score:',rmse)
    print('RMSLE score:',(mean_squared_log_error(y_test,abs(y_pred))))
    print("-----------------------------------------")

def cross_validate(model,x,y,n=3):
    '''
    Returns cross validation score
    example usage:
            cross_validate
    '''
    score = cross_val_score(model, X, y, scoring='accuracy',n_jobs=-1, cv=n)
    cv_score = (round(np.mean(score),5) * 100)
    print('Cross-validation Score >>',cv_score)

def roc_curve_graph(y_test, y_pred):
    '''
    Plots Area Under Curve graph
    #### only works for binary classification
    example usage:
            roc_curve_graph(actual-values,predicted-values)
    '''
    fpr, tpr, threshold = metrics.roc_curve(y_test, y_pred)
    roc_auc = metrics.auc(fpr, tpr)
    plt.title('Receiver Operating Characteristic')
    plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)
    plt.legend(loc = 'lower right')
    plt.plot([0, 1], [0, 1],'r--')
    plt.xlim([0, 1])
    plt.ylim([0, 1])
    plt.ylabel('True Positive Rate')
    plt.xlabel('False Positive Rate')
    plt.show()

#### Basic Machine learning Models
def linear_reg(X_train,y_train,X_test,y_test):
    '''
    performs Logistic Regression Algorithm and returns predicted values and model
    usage example:
            y_pred, model = lgb_reg(X_train,y_train,X_test,y_test)
    '''
    lr = LinearRegression()
    lr.fit(X_train,y_train)
    y_pred = lr.predict(X_test)
    return y_pred, lr

def logistic_reg(X_train,y_train,X_test,y_test):
    '''
    performs Linear Regression Algorithm and returns predicted values and model
    usage example:
            y_pred, model = lgb_reg(X_train,y_train,X_test,y_test)
    '''
    lr = LogisticRegression()
    lr.fit(X_train,y_train)
    y_pred = lr.predict(X_test)
    return y_pred, lr

def randomforest_classifier(X_train,y_train,X_test,y_test):
    '''
    Performs Random Forest Algorithm and returns predicted values and model and plot feature importance graph
    usage example:
            y_pred, model = lgb_reg(X_train,y_train,X_test,y_test)
    '''
    rfc = RandomForestClassifier(n_estimators = 200,random_state=101,class_weight='balanced')
    rfc.fit(X_train,y_train)
    y_pred = rfc.predict(X_test)
    try:
        #### plotting feature importance
        importances=rfc.feature_importances_
        feature_importances=pd.Series(importances, index=X_train.columns).sort_values(ascending=False)
        plt.figure(figsize=(10,7))
        sns.barplot(x=feature_importances[0:10], y=feature_importances.index[0:10])
        plt.title('Random Forest Feature Importance',size=20)
        plt.ylabel("Features")
        plt.show()
    except:
        print('Unable to plot Feature Importance Graph')
    return y_pred,rfc

def randomforest_reg(X_train,y_train,X_test,y_test):
    '''
    Performs Random Forest Algorithm and returns predicted values and model and plot feature importance graph
    usage example:
            y_pred, model = lgb_reg(X_train,y_train,X_test,y_test)
    '''
    rfc = RandomForestRegressor(n_estimators=100,max_depth=8, random_state=101)
    rfc.fit(X_train,y_train)
    y_pred = rfc.predict(X_test)
    try:
        #### plotting feature importance
        importances=rfc.feature_importances_
        feature_importances=pd.Series(importances, index=X_train.columns).sort_values(ascending=False)
        plt.figure(figsize=(10,7))
        sns.barplot(x=feature_importances[0:10], y=feature_importances.index[0:10])
        plt.title('Random Forest Feature Importance',size=20)
        plt.ylabel("Features")
        plt.show()
    except:
        print('Unable to plot Feature Importance Graph')
    return y_pred, rfc

def xgb_classifier(X_train,y_train,X_test,y_test):
    '''
    Performs XGBoost Algorithm and returns predicted values and model and plot feature importance graph
    usage example:
            y_pred, model = xgb_classifier(X_train,y_train,X_test,y_test)
    '''
    xgb = XGBClassifier(n_estimator=100,max_depth=8,class_weight='balanced',refit='AUC')
    xgb.fit(X_train,y_train)
    y_pred = xgb.predict(X_test)
    return y_pred, xgb

def xgb_reg(X_train,y_train,X_test,y_test):
    '''
    Performs XGBoost Algorithm and returns predicted values and model and plot feature importance graph
    usage example:
            y_pred, model = xgb_reg(X_train,y_train,X_test,y_test)
    '''
    model_xgb = XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
                             learning_rate=0.05, max_depth=3,
                             min_child_weight=1.7817, n_estimators=2200,
                             reg_alpha=0.4640, reg_lambda=0.8571,
                             subsample=0.5213, silent=1,
                             random_state =42, nthread = -1)
    xgb.fit(X_train,y_train)
    y_pred = xgb.predict(X_test)
    return y_pred, model_xgb

def lgb_classifier(X_train,y_train,X_test,y_test):
    '''
    Performs LGBM boost Algorithm and returns predicted values and model and plot feature importance graph
    usage example:
            y_pred, model = lgb_classifier(X_train,y_train,X_test,y_test)
    '''
    y_train1 = y_train.values
    model = lgb.LGBMClassifier(n_estimators=600,random_state=101,max_depth=8,
                               verbose=1,class_weight='balanced')
    model.fit(X_train, y_train1)
    y_pred = model.predict(X_test)
    try:
        #### plotting feature importance
        fig, ax = plt.subplots(figsize=(12,8))
        lgb.plot_importance(model, max_num_features=10, height=0.8, ax=ax)
        ax.grid(False)
        plt.title("LightGBM - Feature Importance", fontsize=15)
        plt.show()
    except:
        print('Unable to plot Feature Importance Graph')
    return y_pred, model

def lgb_reg(X_train,y_train,X_test,y_test):
    '''
    Performs LGBM boost Algorithm and returns predicted values and model and plot feature importance graph
    usage example:
            y_pred, model = lgb_reg(X_train,y_train,X_test,y_test)
    '''
    model = lgb.LGBMRegressor(n_estimators=600,random_state=101,max_depth=8,
                               verbose=1)
    model.fit(X_train, y_train)
    y_pred = model.predict(X_test)
    try:
        #### plotting feature importance
        fig, ax = plt.subplots(figsize=(12,8))
        lgb.plot_importance(model, max_num_features=10, height=0.8, ax=ax)
        ax.grid(False)
        plt.title("LightGBM - Feature Importance", fontsize=15)
        plt.show()
    except:
        print('Unable to plot Feature Importance Graph')
    return y_pred, model

def adaboost_classifier(X_train,y_train,X_test,y_test):
    '''
    Performs Ada Boost Algorithm and returns predicted values and model and plot feature importance graph
    usage example:
            y_pred, model = adaboost_classifier(X_train,y_train,X_test,y_test)
    '''
    ada = AdaBoostClassifier(n_estimators=100)
    ada.fit(X_train,y_train)
    y_pred = ada.predict(X_test)
    try:
        importances=ada.feature_importances_
        feature_importances=pd.Series(importances, index=X_train.columns).sort_values(ascending=False)
        plt.figure(figsize=(10,7))
        sns.barplot(x=feature_importances[0:10], y=feature_importances.index[0:10])
        plt.title('AdaBoost Feature Importance',size=20)
        plt.ylabel("Features")
        plt.show()
    except:
        print('Unable to plot Feature Importance Graph')
    return y_pred, ada

def adaboost_reg(X_train,y_train,X_test,y_test):
    '''
    Performs Ada Boost Algorithm and returns predicted values and model and plot feature importance graph
    usage example:
            y_pred, model = adaboost_reg(X_train,y_train,X_test,y_test)
    '''
    ada = AdaBoostRegressor(n_estimators=100, random_state=0)
    ada.fit(X_train,y_train)
    y_pred = ada.predict(X_test)
    try:
        importances=ada.feature_importances_
        feature_importances=pd.Series(importances, index=X_train.columns).sort_values(ascending=False)
        plt.figure(figsize=(10,7))
        sns.barplot(x=feature_importances[0:10], y=feature_importances.index[0:10])
        plt.title('AdaBoost Feature Importance',size=20)
        plt.ylabel("Features")
        plt.show()
    except:
        print('Unable to plot Feature Importance Graph')
    return y_pred, ada

def Gradient_boost_classifier(X_train,y_train,X_test,y_test):
    '''
    Performs Gradient Boost Algorhtm and returns predicted values and model
    usage example:
            y_pred, model = Gradient_boost_reg(X_train,y_train,X_test,y_test)
    '''
    GBoost = GradientBoostingClassifier(n_estimators = 3000, max_depth = 5,max_features='sqrt',
                                            min_samples_split = 10,learning_rate = 0.005,
                                            min_samples_leaf=15,random_state =10)
    GBoost.fit(X_train, y_train)
    y_pred = GBoost.predict(X_test)
    return y_pred, GBoost

def Gradient_boost_reg(X_train,y_train,X_test,y_test):
    '''
    Performs Gradient Boost Algorhtm and returns predicted values and model
    usage example:
            y_pred, model = Gradient_boost_reg(X_train,y_train,X_test,y_test)
    '''
    GBoost = GradientBoostingRegressor(n_estimators = 3000, max_depth = 5,max_features='sqrt',
                                            min_samples_split = 10,learning_rate = 0.005,loss = 'huber',
                                            min_samples_leaf=15,random_state =10)
    GBoost.fit(X_train, y_train)
    y_pred = GBoost.predict(X_test)
    return y_pred, GBoost

def grid_search(model,flag,X_train,y_train,X_test,y_test):
    '''
    Performs GridSearchCV when model and parameters passed and return predicted values and best_estimaor parameters
    usage example:
            y_pred, model = grid_search(model,flag,x_train,y_train,X_test,y_test)
        model - pass any machine learning model
        flag:
            1. 'quick' (get quicky executed)
            2. 'deep' (takes time)
        X_train,y_train,X_test,y_test generated from train_test_split

    '''
    if flag == 'quick':
        params = {
            #'bootstrap': [True],
            'max_depth': [80, 90, 100, 110],
            'min_samples_leaf': [3, 4, 5],
            'min_samples_split': [8, 10, 12],
            'n_estimators': [20, 30, 40, 50],
        }
    elif flag == 'deep':
        params = {'min_child_weight':[5,6,7], 'gamma':[i/10.0 for i in range(3,7)],
                  'subsample':[i/10.0 for i in range(6,11)],
                  'colsample_bytree':[i/10.0 for i in range(6,11)], 'max_depth': [5,6,7,8]}

    print('Performing GridSearchCV \nPlease wait............')
    grid = GridSearchCV(model, params)
    grid.fit(X_train,y_train)
    y_pred = grid.best_estimator_.predict(X_test)
    print('Done  ¯\_(ツ)_/¯\n')
    print('best parameters: \n', grid.best_estimator_)
    return y_pred, grid

def kmeans(dataframe,n):
    '''
    Performs Kmeans clustering Algorithm
    return preidicted clusters when passed data and no of required clusters
    example usage:
            clusters = (dataframe,n)
        n- no of required clusters
    '''
    km = KMeans(n_clusters=n)
    y_pred_c = km.fit_predict(dataframe)
    return y_pred_c

def classification(X,y):
    '''
This Method Provides Menu based Algorithm Selection (Classification) for Prediction,It return predicted values(y_pred) and machine learning model for further usage.
    example usage:
             y_pred, model = classification(X,y)
    X - features
    y - target
    ### just pass cleaned X and y
    '''
    X_train,X_test,y_train,y_test = train_test_split(X,y,train_size=.7,random_state=42)
    print("########## MENU ##############\n")
    print('1. Logisitic Regression \n2. Random Forest \n3. XGB Classifier \n4. LGB CLassifier \n5. Gradient Boosting Classifier \n6. AdaBoost Classifier\n')
    x = int(input('Input No, which you want:'))
   # if x == 99:
     #   return quick_pred(X,y,'c')
    if x == 1:
        print('Predicting with Logisitic Regression Classifier')
        y_pred, model = logistic_reg(X_train,y_train,X_test,y_test)
    elif x == 2:
        print('Predicting with Random Forest Classifier')
        y_pred, model = randomforest_classifier(X_train,y_train,X_test,y_test)
    elif x == 3:
        print('Predicting with XGB boost Classifier')
        y_pred, model = xgb_classifier(X_train,y_train,X_test,y_test)
    elif x == 4:
        print('Predicting with Light GBM Classifier')
        y_pred, model = lgb_classifier(X_train,y_train,X_test,y_test)
    elif x == 5:
        print('Predicting with Gradient Bossting Classifier')
        y_pred, model = Gradient_boost_classifier(X_train,y_train,X_test,y_test)
    elif x == 6:
        print('Predicting with Ada Boost CLassifier')
        y_pred, model = adaboost_classifier(X_train,y_train,X_test,y_test)
    else:
        print('\n\n Please input appropirate no (*-*)')
        classification(X_train,y_train,X_test,y_test)

    classification_result(y_test,y_pred)
    print('\nModel used:',model,'\n Done.')
    return y_pred,model

def regression(X,y):
    '''
This Method Provides Menu based Algorithm Selection (Regression) for Prediction, It return predicted values(y_pred) and machine learning model for further usage.
    example usage:
             y_pred, model = classification(X,y)
    X - features
    y - target
    ### just pass cleaned X and y
    '''
    X_train,X_test,y_train,y_test = train_test_split(X,y,train_size=.7,random_state=42)
    print("########## MENU ##############\n")
    print('1. linear Regression \n2. Random Forest regressor \n3. XGB regressor \n4. LGB regressor \n5. Gradient Boosting regressor \n6.AdaBoost regressor\n')
    x = int(input('Input No, which you want:'))
    #if x == 99:
     #   return quick_pred(X,y,'r')
    if x == 1:
        print('Predicting with linear Regression')
        y_pred, model = linear_reg(X_train,y_train,X_test,y_test)
    elif x == 2:
        print('Predicting with Random Forest Regressor')
        y_pred, model = randomforest_reg(X_train,y_train,X_test,y_test)
    elif x == 3:
        print('Predicting with XGB boost Regressor')
        y_pred, model = xgb_reg(X_train,y_train,X_test,y_test)
    elif x == 4:
        print('Predicting with Light GBM Regressor')
        y_pred, model = lgb_reg(X_train,y_train,X_test,y_test)
    elif x == 5:
        print('Predicting with Gradient Bossting Regressor')
        y_pred, model = Gradient_boost_reg(X_train,y_train,X_test,y_test)
    elif x == 6:
        print('Predicting with Ada Boost Regressor')
        y_pred, model = adaboost_classifier(X_train,y_train,X_test,y_test)
    else:
        print('\n\n Please input appropirate no (*-*)')
        regression(X_train,y_train,X_test,y_test)

    regression_result(y_test,y_pred)
    print('\nModel used:',model,'\n Done.')
    return y_pred,model

## Quick Machine learning

### please to input only cleaned dataframe
### specify "r" for regression and 'c' for classification
### X are features, y is target
def clean_data(dataframe):
    dataframe.apply(lambda x: x.replace(" ",np.nan,inplace=True))
    dataframe = drop_columns(dataframe,20)
    nullcolumns = []
    for i,j in dict(dataframe.isnull().sum()).items():
        if j > 0:
            nullcolumns.append(i)

    num1, cat1 = getcolumnstype(dataframe[nullcolumns])
    dataframe = fillnulls(dataframe,'unknown',cat1)
    dataframe = fillnulls(dataframe,'median',num1)
    return dataframe

def quick_pred(X,y,flag):
    '''
    Automatically Trains & Tests data using Many Famous Machine learning Algrithms and return scores in colorized dataframe table format.
    **** Passed data should be cleaned *****
    #### use quick_ml() function for uncleaned data
example:
        quick(X,y,flag)
    * X: Specify features
    * y: spcifigy Target column
    * flag: specify 'r'for regression & 'c' for classification
    '''
    #ss = StandardScaler()
    #X = ss.fit_transform(X)
    X_train,X_test,y_train,y_test = train_test_split(X,y,random_state=101,test_size=30)
    print('\nPlease wait Training-Testing with all models..\n')
    if flag.lower() == 'r':
        models = {"LinearRegression": LinearRegression(),
                  "RandomForestRegression": RandomForestRegressor(),
                  "XGBoostRegressor":XGBRegressor(),
                  "LGBoostRegressor":LGBMRegressor(),
                  "AdaBoostRegressor":AdaBoostRegressor(),
                  "SupportVectorMachine":SVR(),
                  "GradientBoostingRegression": GradientBoostingRegressor()}
        mn,r2score,rmse,rmsle,mse = [],[],[],[],[]

        for model_name, model in tqdm(models.items()):
            model.fit(X_train, y_train)
            pred = model.predict(X_test)
            mn.append(model_name)
            r2score.append(r2_score(y_test,pred))
            rmse.append(np.sqrt(mean_squared_error(y_test,pred)))
            rmsle.append(np.sqrt(mean_squared_log_error(y_test,abs(pred))))
            mse.append(mean_squared_error(y_test,pred))
            print('Done with ', model_name)

        results = pd.DataFrame()
        results['Model'] = mn
        results['R^2 score'] = r2score
        results['Root Mean Squared Error'] = rmse
        results['Root Mean Squared Log Error'] = rmsle
        results['Mean Squared Error'] = mse
        results = results.sort_values(by=['R^2 score'],ascending=False)
        return results.style.background_gradient()

    elif flag.lower() == 'c':
        models = {"LogisticRegression": LogisticRegression(),
                      "RandomForestClassifier": RandomForestClassifier(),
                      "XGBoost Classifier":XGBClassifier(),
                      "LGBoost Classifier":LGBMClassifier(),
                      "AdaBoost Classifier":AdaBoostClassifier(),
                      #"SupportVectorMachine":SVC(),
                      "GradientBoostingClassifier": GradientBoostingClassifier()}
        f1,auc,mn,acc,ll = [],[],[],[],[]
        for model_name, model in tqdm(models.items()):
                model.fit(X_train, y_train)
                pred = model.predict(X_test)
                mn.append(model_name)
                f1.append(f1_score(y_test,pred,average = "weighted"))
                if len(pd.Series(pred).value_counts()) == 2:
                    auc.append(roc_auc_score(y_test,pred))
                else:
                    auc.append(multiclass_roc_auc_score(y_test, pred))
                acc.append(accuracy_score(y_test, pred))
                ll.append(balanced_accuracy_score(y_test,pred))
                print('Done with ', model_name)

        results = pd.DataFrame()
        results['Model'] = mn
        results['F1 score'] = f1
        results['AUC-ROC score'] = auc
        results['Accuracy'] = acc
        results['Balanced Accuracy Score'] = ll
        results = results.sort_values(by=['F1 score'],ascending=False)
        print(results)
        print('\nCheck returned table\n')
        return results.style.background_gradient()

### pass Dataframe,Target column-name & flag (r for regression and c for classification)
def quick_ml(df1,target,flag,n=0):
    '''
quick_ml: No hazzle Machine learning.
    This methods Automatically takes dataframe and target column and flag as input and gives you the Preidiction using famous Machine learning Algorithm.
example usage:-
            result, model = quick_ml(dataframe,'target-column-name','flag')
parameters:
 * dataframe: the dataframe you wan't to get prediction from
 * 'target-column-name' : specify target column name
 * 'flag': specify flag 'r'for regression and 'c'for classification
optional:
  'n'= no of features you wan't to be used for prediction using RFE
      (by default set to 0 and predicts using all features )
      (Rfe May Take Time in Execution so wait)

    '''
    info(df1)
    if df1.isnull().sum().sum() > 0:
        df1 = clean_data(df1)

    num,cat = getcolumnstype(df1)
    if len(cat) > 0:
        df1=label_encode(df1)
    corr_heatmap(df1,'interactive')
    X = df1.drop(target,axis=1)
    y = df1[target]
    X_train,X_test,y_train,y_test = train_test_split(X,y,train_size=.7,random_state=42)

    if flag == 'c':
        showbias(df1,target)
        print('Detailed Executing with RandomForest ..')
        y_pred, model = randomforest_classifier(X_train,y_train,X_test,y_test)
        classification_result(y_test,y_pred)
        if n>0:
            print('\n please wait while performing prediction using RFE...')
            rfecols = rfe(model,X_train,y_train,n)
            X_train = X_train[rfecols]
            X_test = X_test[rfecols]
            print('RFE Selected Features',rfecols)
            y_pred, model = randomforest_classifier(X_train,y_train,X_test,y_test)
        print('Features used for prediction\n',X_train.columns)
        X = scale(X)
        return quick_pred(X,y,'c'),model

    elif flag == 'r':
        box_hist_plot(df1,target)
        print('Detailed Executing with RandomForest ..')
        y_pred, model = randomforest_reg(X_train,y_train,X_test,y_test)
        regression_result(y_test,y_pred)
        if n>0:
            print('\n please wait while performing prediction using RFE...')
            rfecols = rfe(model,X_train,y_train,n)
            X_train = X_train[rfecols]
            X_test = X_test[rfecols]
            y_pred, model = randomforest_reg(X_train,y_train,X_test,y_test)
            print(' RFE Selected Features',rfecols)
        print('Features used for prediction:\n',X_train.columns)
        X = scale(X)
        return quick_pred(X,y,'r'),model

### Visualization

def showbias(dataframe,target):
    '''
    Plots interactive Pie Graph for Column and show percentage of each class in column
    example usage:-
      showbias(dataframe,'columname')
    '''
    data = dataframe.copy()
    labels = list(data[target].value_counts().index)
    fig = px.pie(names = labels,values = data[target].value_counts().values,title='Percentage data of '+target)
    fig.update_traces(hole=.4, hoverinfo="label+percent+name")
    fig.show()

def corr_heatmap(dataframe,style=None):
    '''
    Plot interactive corealtion heatmap for dataframe passed.
    example usage:-
      corr_heatmap(dataframe,style)
      optional style:
        1. 'basic' - return basic tabular highlighted corelation table
        2. interactive - plots interactive(zoomable) heatmap with plotly

    if columns names are not passed it will plot histrogram and boxplot for all numeric columns
    '''
    if style == 'basic':
        return dataframe.corr().style.background_gradient()
    elif style =='interactive':
        corrs = dataframe.corr()
        figure = ff.create_annotated_heatmap(
            z=corrs.values,
            x=list(corrs.columns),
            y=list(corrs.index),
            annotation_text=corrs.round(2).values,
            showscale=True)
        figure.show()
    else:
        plt.figure(figsize=(20,20))
        sns.heatmap(dataframe.corr(),square=True,annot=True,linewidths=0.2,cmap='Greens')

def box_hist_plot(dataframe,*argv):
    '''
    Plot interactive Boxplot & Histogram for n no of column passed after dataframe name.
    example usage:-
        box_hist_plot(dataframe,'columname1','columname2')

    if columns names are not passed it will plot histrogram and boxplot for all numeric columns
    '''
    columns = list(argv)
    if len(columns) > 0:
        for i in columns:
            dataframe[i].iplot(kind='hist', title= i+' Distribution')
            dataframe[i].iplot(kind='box', title= i+' Box plot')
    else:
        for i in dataframe._get_numeric_data().columns:
            dataframe[i].iplot(kind='hist', title= i+' Distribution')
            dataframe[i].iplot(kind='box', title= i+' Box plot')

def confusion_mat(y_test,y_pred,cmap=None):
    '''
    plots confusion matrix
    '''
    if cmap != None:
        clr = cmap
    else:
        clr = 'gnuplot2_r'
    y_pred_temp = pd.Series(y_pred)
    labels = unique_labels(y_test,y_pred)
    sns.heatmap(confusion_matrix(y_test,y_pred),annot=True,
                fmt='d',cmap=clr,
                cbar=False,
                xticklabels = labels,
                yticklabels = labels,
                linewidth=0.2,
                linecolor='black'
               )
    plt.ylabel('Actual',size=15)
    plt.xlabel('Predicted',size=15)
    plt.title('Confusion Matrix',size=20)
    plt.show()

from wordcloud import WordCloud
def wordcloud(df1,columname,bgcolor=None):
    '''
    df : dataframe
    columname : columne for which word cloud needed
    bgcolor : black or white background
    '''
    if bgcolor != None:
        bgcolor = bgcolor
    from wordcloud import WordCloud
    x2011 = df1[columname]
    plt.subplots(figsize=(8,8))
    wordcloud = WordCloud(
                              background_color=bgcolor,
                              width=512,
                              height=384
                             ).generate(" ".join(x2011))
    plt.imshow(wordcloud)
    plt.axis('off')
    plt.show()

### Visualization

def showbias(dataframe,target):
    '''
    Plots interactive Pie Graph for Column and show percentage of each class in column
    example usage:-
      showbias(dataframe,'columname')
    '''
    data = dataframe.copy()
    labels = list(data[target].value_counts().index)
    fig = px.pie(names = labels,values = data[target].value_counts().values,title='Percentage data of '+target)
    fig.update_traces(hole=.4, hoverinfo="label+percent+name")
    fig.show()

def corr_heatmap(dataframe,style=None):
    '''
    Plot interactive corealtion heatmap for dataframe passed.
    example usage:-
      corr_heatmap(dataframe,style)
      optional style:
        1. 'basic' - return basic tabular highlighted corelation table
        2. interactive - plots interactive(zoomable) heatmap with plotly

    if columns names are not passed it will plot histrogram and boxplot for all numeric columns
    '''
    if style == 'basic':
        return dataframe.corr().style.background_gradient()
    elif style =='interactive':
        corrs = dataframe.corr()
        figure = ff.create_annotated_heatmap(
            z=corrs.values,
            x=list(corrs.columns),
            y=list(corrs.index),
            annotation_text=corrs.round(2).values,
            showscale=True)
        figure.show()
    else:
        plt.figure(figsize=(20,20))
        sns.heatmap(dataframe.corr(),square=True,annot=True,linewidths=0.2,cmap='Greens')

def box_hist_plot(dataframe,*argv):
    '''
    Plot interactive Boxplot & Histogram for n no of column passed after dataframe name.
    example usage:-
        box_hist_plot(dataframe,'columname1','columname2')

    if columns names are not passed it will plot histrogram and boxplot for all numeric columns
    '''
    columns = list(argv)
    if len(columns) > 0:
        for i in columns:
            dataframe[i].iplot(kind='hist', title= i+' Distribution')
            dataframe[i].iplot(kind='box', title= i+' Box plot')
    else:
        for i in dataframe._get_numeric_data().columns:
            dataframe[i].iplot(kind='hist', title= i+' Distribution')
            dataframe[i].iplot(kind='box', title= i+' Box plot')

def confusion_mat(y_test,y_pred,cmap=None):
    '''
    plots confusion matrix
    '''
    if cmap != None:
        clr = cmap
    else:
        clr = 'gnuplot2_r'
    y_pred_temp = pd.Series(y_pred)
    labels = unique_labels(y_test,y_pred)
    sns.heatmap(confusion_matrix(y_test,y_pred),annot=True,
                fmt='d',cmap=clr,
                cbar=False,
                xticklabels = labels,
                yticklabels = labels,
                linewidth=0.2,
                linecolor='black'
               )
    plt.ylabel('Actual',size=15)
    plt.xlabel('Predicted',size=15)
    plt.title('Confusion Matrix',size=20)
    plt.show()

from wordcloud import WordCloud
def wordcloud(df1,columname,bgcolor=None):
    '''
    df : dataframe
    columname : columne for which word cloud needed
    bgcolor : black or white background
    '''
    if bgcolor != None:
        bgcolor = bgcolor
    from wordcloud import WordCloud
    x2011 = df1[columname]
    plt.subplots(figsize=(8,8))
    wordcloud = WordCloud(
                              background_color=bgcolor,
                              width=512,
                              height=384
                             ).generate(" ".join(x2011))
    plt.imshow(wordcloud)
    plt.axis('off')
    plt.show()

### NLP processing

from textblob import TextBlob ### For Sentiment Polarity
stop_words = ['has', 'no', 'doing', "weren't", 'have', 'while', 'an', 'some', 'above', 'than', "mightn't", 'all',
              'theirs', 'am', 'are', 'won', 'does', 'until', 'again', "wasn't", 'will', 'was', 'not', 'into', 'so',
              'that', 'my', 'before', 'itself', 'whom', 'for', 'we', 'do', 'just', "doesn't", 'were', 'during', 'them',
              'be', 'from', 'there', 'after', 'any', 'did', 'once', "needn't", 'needn', 't', "wouldn't", 'your', 'in', 'they', 'his', 'had',
              'wasn', 'hers', 'd', 'why', 'being', 'yourselves', 'i', 'few', 'our', 'me', 'himself', 'shan', 'which', "hasn't", 'by',
              'at', 'wouldn', "that'll", "shan't", 'm', "you've", 'if', 'ain', 'further', 'hasn', 're', 'its', 'herself', 'haven', 'here',
              'themselves', 'yourself', 'didn', 'been', 'where', 'each', 'because', 'hadn', 'on', "couldn't", 'mightn', 'having', 'isn', 'don', 'their',
              'he', "hadn't", 'more', 'such', "shouldn't", 'she', 'can', 'as', "won't", 'who', 'now', "you'll", 'between', 'other', 'very', 'ours', 'and', 'her', 'it', 'weren',
              'most', "aren't", "didn't", "isn't", 'when', 'shouldn', 'you', 'under', 'below', 'how', 'is',
              "should've", 'through', 've', 'doesn', 'same', 'should', "it's", "she's", "don't", 'couldn', 'or', 'to', 'those', 'this', 'with', 'both', 'too', 'of', 'him', "mustn't", 'what',
              'nor', 'own', 'o', 's', 'up', 'myself', 'll', 'about', 'y', 'yours', 'the', 'aren', 'a', 'over',
              'against', "you're", 'ourselves', 'then', 'down', 'mustn', 'off', 'only', 'ma', "you'd", 'but', 'these', 'out', "haven't"]
import string
import nltk
from nltk.stem import PorterStemmer,WordNetLemmatizer
from sklearn.feature_extraction.text import TfidfVectorizer
from tqdm.notebook import tqdm as tqdm

def text_process(mess):
    '''
    Method removes Punctuations and Special Characters from column
    and return lower cased
    '''
    nopunc = [char for char in mess if char not in string.punctuation]
    nopunc = ''.join(nopunc)
    nopunc =  ' '.join([word for word in nopunc.split() if word.lower() not in stop_words])
    return str(TextBlob(nopunc)).lower()

def nlp_text(dataframe,columname):
    '''
    Method converts Textual data of Column to Tfidf vector after:
    Removing Punctutions, Speical Symbol/character, Coverts Uppercase to lowercase and Lemmetization
    ### only column passed is converted to vector not whole dataframe
    ### need to manually join output data vector with original dataframe
    example usage:
            data = nlp_text(dataframe,'columname')
    ## after this execute this :
        dataframe.drop('columname',axis=1)
        newdf = pd.concat([dataframe,pd.DataFrame(data)],axis=1)
    '''
    joined = dataframe[columname]
    joined.replace("[^a-zA-Z]"," ",regex=True,inplace=True)
    joined = joined.apply(text_process)

    story = []
    for i in joined:
        story.append(i)

    stemmer = PorterStemmer()
    lemmatizer = WordNetLemmatizer()
    new_sentence = []
    for i in tqdm(range(len(story))):
        words = nltk.word_tokenize(story[i])
        words = [lemmatizer.lemmatize(word) for word in words]
        new_sentence.append(' '.join(words))

        ## just checking result with both, Using Lemmatization as it is good.
        #for word in words:
            #print(word,"---to setm-------->",stemmer.stem(word))
            #print(word,'---to lematize---->',lemmatizer.lemmatize(word))
    print('Processed Text looks like >>\n',new_sentence[2])

    tfid = TfidfVectorizer()
    data = tfid.fit_transform(new_sentence).toarray()
    print('\nshape before Text Processing:',dataframe[columname].shape)
    print('\nshape before Text Processing:',data.shape)
    return data

### Deep Learning
from sklearn.neural_network import MLPClassifier, MLPRegressor
def neural_network(X_train,y_train,X_test,y_test,flag,solver=None):
    '''
    example usage:
        y_pred,model = neural_network(X_train,y_train,X_test,y_test,flag)
        flag: 'r' for regresssion ,'c' for classification
        optional solver: 'adam','lbfgs','sgd'

    Neural Netowrks needs to be hardcoded for every dataset
    try optimizing them:
        solver = 'adam','lbfgs','sgd'
        hidden_layer_sizes=(100, 100),
        solver='adam',tol=1e-2,
        max_iter=500, random_state=1
    '''
    print('Please wait, Building Neural Netowrk....')
    if solver != None:
        bp = solver
    else:
        bp = 'lbfgs'

    if flag == 'c':
        mlp = make_pipeline(StandardScaler(),
                            MLPClassifier(solver = bp))
        mlp.fit(X_train, y_train)
        y_pred = mlp.predict(X_test)
        classification_result(y_test,y_pred)
    elif flag =='r':
        #hidden_layer_sizes=(100, 100),solver='adam',tol=1e-2, max_iter=500, random_state=1
        mlp = make_pipeline(StandardScaler(),
                            MLPRegressor(solver = bp))
        mlp.fit(X_train, y_train)
        y_pred = mlp.predict(X_test)
        regression_result(y_test,y_pred)
    else:
        print('Please Specify Flag (r) or (c)')

    return y_pred, mlp

from sklearn.neural_network import MLPClassifier, MLPRegressor
def neural_network(X_train,y_train,X_test,y_test,flag,solver=None):
    '''
    example usage:
        y_pred,model = neural_network(X_train,y_train,X_test,y_test,flag)
        flag: 'r' for regresssion ,'c' for classification
        optional solver: 'adam','lbfgs','sgd'

    Neural Netowrks needs to be hardcoded for every dataset
    try optimizing them:
        solver = 'adam','lbfgs','sgd'
        hidden_layer_sizes=(100, 100),
        solver='adam',tol=1e-2,
        max_iter=500, random_state=1
    '''
    print('Please wait, Building Neural Netowrk....')
    if solver != None:
        bp = solver
    else:
        bp = 'lbfgs'

    if flag == 'c':
        mlp = make_pipeline(StandardScaler(),
                            MLPClassifier(solver = bp))
        mlp.fit(X_train, y_train)
        y_pred = mlp.predict(X_test)
        classification_result(y_test,y_pred)
    elif flag =='r':
        #hidden_layer_sizes=(100, 100),solver='adam',tol=1e-2, max_iter=500, random_state=1
        mlp = make_pipeline(StandardScaler(),
                            MLPRegressor(solver = bp))
        mlp.fit(X_train, y_train)
        y_pred = mlp.predict(X_test)
        regression_result(y_test,y_pred)
    else:
        print('Please Specify Flag (r) or (c)')

    return y_pred, mlp

# In[14]:

### Deployment

import pickle as pickle

app_py = ['import numpy as np\n',
 'from flask import Flask, request, jsonify, render_template\n',
 'import pickle\n',
 '\n',
 'app = Flask(__name__)\n',
 "model = pickle.load(open('model.pkl', 'rb'))\n",
 '\n',
 "@app.route('/')\n",
 'def home():\n',
 "    return render_template('index.html')\n",
 '\n',
 "@app.route('/predict',methods=['POST'])\n",
 'def predict():\n',
 "    '''\n",
 '    For rendering results on HTML GUI\n',
 "    '''\n",
 '    int_features = [int(x) for x in request.form.values()]\n',
 '    final_features = [np.array(int_features)]\n',
 '    prediction = model.predict(final_features)\n',
 '\n',
 '    output = round(prediction[0], 2)\n',
 '\n',
 "    return render_template('index.html', prediction_text='Prediction:  {}'.format(output))\n",
 '\n',
 "@app.route('/predict_api',methods=['POST'])\n",
 'def predict_api():\n',
 "    '''\n",
 '    For direct API calls trought request\n',
 "    '''\n",
 '    data = request.get_json(force=True)\n',
 '    prediction = model.predict([np.array(list(data.values()))])\n',
 '\n',
 '    output = prediction[0]\n',
 '    return jsonify(output)\n',
 '\n',
 'if __name__ == "__main__":\n',
 '    app.run(debug=True)']

### Makes Html file for deployment
def index_file(title):
    htm = ['<!doctype html>\n',
     '<html lang="en">\n',
     '<head>\n',
     '    <meta charset="UTF-8">\n',
     '    <title>Ml Deploy</title>\n',
     '    <style>\n',
     '        body{\n',
     '            margin: 0;\n',
     '            padding: 0;\n',
     '            }\n',
     '        body:before{\n',
     "            content: '';\n",
     '            position: fixed;\n',
     '            width: 100vw;\n',
     '            height: 100vh;\n',
     '            background: -webkit-linear-gradient(rgb(92, 57, 219),rgb(173, 218, 210),rgb(235, 193, 56)); \n',
     '            background: -o-linear-gradient(rgb(92, 57, 219),rgb(173, 218, 210),rgb(235, 193, 56)); \n',
     '            background: -moz-linear-gradient(rgb(92, 57, 219),rgb(173, 218, 210),rgb(235, 193, 56)); \n',
     '            background: linear-gradient(rgb(92, 57, 219), rgb(173, 218, 210),rgb(235, 193, 56)); \n',
     '            background-color:rgb(46, 236, 157); \n',
     '            background-position: center center;\n',
     '            background-repeat: no-repeat;\n',
     '            background-attachment: fixed;\n',
     '            background-size: relative;\n',
     '            -webkit-filter: blur(10px);\n',
     '            -moz-filter: blur(10px);\n',
     '            -o-filter: blur(10px);\n',
     '            -ms-filter: blur(10px);\n',
     '            filter: blur(10px);\n',
     '        }\n',
     '        .form\n',
     '        {\n',
     '            position: absolute;\n',
     '            top:5%;\n',
     '            left:30%;\n',
     '            margin-bottom: 20px;\n',
     '            align-self :center;\n',
     '            /*transform: translate(-50%,-50%);*/\n',
     '            width: 400px;\n',
     '            height: relative;\n',
     '            padding: 50px 40px;\n',
     '            box-sizing: border-box;\n',
     '            background: rgba(0,0,0,.55);\n',
     '        \n',
     '        }\n',
     '        .form h2 {\n',
     '            margin: 0;\n',
     '            padding: 0 0 20px;\n',
     '            color:white;\n',
     "            font-family:'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;\n",
     '            text-align: center;\n',
     '            text-transform: uppercase;\n',
     '        }\n',
     '        .form p\n',
     '        {\n',
     '            margin: 0;\n',
     '            padding: 0;\n',
     '            font-weight: bold;\n',
     '            color: #fff;\n',
     '        }\n',
     '        .form input\n',
     '        {\n',
     '            width: 100%;\n',
     '            margin-bottom: 20px;\n',
     '        }\n',
     '        .form input[type="text"]\n',
     '        {\n',
     '            border: none;\n',
     '            border-bottom: 1px solid #fff;\n',
     '            background: transparent;\n',
     '            outline: none;\n',
     '            height: 40px;\n',
     '            color: #fff;\n',
     '            font-size: 16px;\n',
     '        }\n',
     '        .form input[type="submit"] {\n',
     '            height: 40px;\n',
     '            color: whitesmoke;\n',
     '            font-weight: bold;\n',
     '            font-size: 15px;\n',
     '            background: rgb(28, 235, 131);\n',
     '            cursor: pointer;\n',
     '            border-radius: 25px;\n',
     '            border: none;\n',
     '            outline: none;\n',
     '            margin-top: 5%;\n',
     '        }\n',
     '        .form a\n',
     '        {\n',
     '            color: #fff;\n',
     '            font-size: 14px;\n',
     '            font-weight: bold;\n',
     '            text-decoration: none;\n',
     '        }\n',
     '        input[type="checkbox"] {\n',
     '            width: 20%;\n',
     '        }\n',
     '        h2{\n',
     '            color:white;\n',
     '            font-family: sans-serif;\n',
     '        }\n',
     '    </style>\n',
     '</head>\n',
     '<body>\n',
     '    <div class="form">\n',
     '        <h2>'+title+'</h2>\n',
     '        <form action="{{ url_for(\'predict\')}}" method="post">\n',
     '            \n',
     '            <input type="submit" name="" value="Predict">\n',
     '        </form>\n',
     '        <form>\n',
     '            <p>{{ prediction_text }}</p>\n',
     '        </form>\n',
     '    </div>\n',
     '</body>\n',
     '</html>']
    return htm

def deploy_one(model,X,title=None):
    '''
    Deploy one Method, Deploys Machine Learning Models to Web App using Flask
    All files will be stored in deployment-files folder created automatically in notebook running directory
    usage:
        deploy_form(model,X,'title')
    model -> Machine Learning Model
    X     -> Features (Note Features should be iterable)
                eg - list,dataframe of Features(X_train)
  'title' -> Title For Html Deplyment UI (optional)
              Default is 'Make Predictions'
    '''
    try:
        if title != None:
            htm_template = index_file(title)
        else:
            htm_template = index_file('Make Predictions')

        os.mkdir('deployment-files')
        os.mkdir('deployment-files/templates')
        pickle.dump(model, open('deployment-files/model.pkl','wb'))
        with open('deployment-files/app.py','w') as mf:
            for k in app_py:
                mf.write(k)
            mf.close()
        with open("deployment-files/templates/index.html", "w") as f1:
            for i in htm_template:
                f1.write(i)
                for k in i.split(' '):
                    if k == '<form':
                        for i in X:
                            htmline = '<input type="text" name="'+str(i)+'" placeholder="'+str(i)+'" required="required" autocomplete="off" />'
                            f1.write(str(htmline)+'\n')
            f1.close()
        print('All Deployment Files Created Successfully\n')
        print('Files Created and stored in : /deployment-files named Folder\n')
        print('Open deployment-files folder and open command prompt and run "python app.py" copy link and paste in browser to run app, enjoy.')
        print('\nFiles created:\n 1. model.pkl - Model file using pickle\n 2.app.py - Flask server file\n3.index.html - User Interface file.')
    except:
        print('There was an error Generating Deployment Files')
        print('Note:\n Delete "Deployment-files" folder if already exists.')

def plotly_browser():
    import IPython
    display(IPython.core.display.HTML('''
            <script src="/static/components/requirejs/require.js"></script>
            <script>
              requirejs.config({
                paths: {
                  base: '/static/base',
                  plotly: 'https://cdn.plot.ly/plotly-latest.min.js?noext',
                },
              });
            </script>
            '''))

### just in case no plots displayed
import plotly.offline as pyo
pyo.init_notebook_mode()


"""
    ИСР 2.1
    На основе кода, предоставленного преподавателем, реализовать генератор чисел ряда Фибоначчи. Генератор требуется создать двумя вариантами: с помощью генератора списков, с помощью функции, внутри которой yield.
"""

from itertools import islice

class FibonacchiLst:
    def __init__(self, max_value):
        self.max_value = max_value
        self.lst = [0, 1]

    def __getitem__(self, idx):
        if self.max_value < 0:
            raise IndexError(idx)

        if idx > 0:
            if self.max_value == 0:
                raise IndexError(idx)

            if idx == 1:
                return self.lst[1]

            if idx == 2 and self.max_value == 1:
                raise IndexError(idx)

            cur_el = self.lst[0] + self.lst[1]
            del self.lst[0]
            self.lst.append(cur_el)

            if cur_el > self.max_value:
                raise IndexError(idx)

            return self.lst[1]

        else:
            return self.lst[0]

def fib():
    a, b = 0, 1
    while True:
        yield a
        a, b = b, a + b

if __name__ == '__main__':
    fib_lst = [num for num in FibonacchiLst(10)]
    print(fib_lst)

    fib_lst = list(islice(fib(), 10))
    print(fib_lst)


from flask import Blueprint, render_template, session, redirect, url_for

import httplib2
import json
from apiclient import discovery
from apiclient.discovery import build
from oauth2client import client

goals = Blueprint('goals', __name__, template_folder='templates')

@goals.route('/')
def index():
    if 'credentials' not in session:
        return redirect(url_for('oauth2callback'))
    credentials = client.OAuth2Credentials.from_json(session['credentials'])
    if credentials.access_token_expired:
        return redirect(url_for('oauth2callback'))

    else:
        goals_id = "0B7e5dn7QCphMU3F4RFcxMkVDSk0"
        http_auth = credentials.authorize(httplib2.Http())
        drive_service = discovery.build('drive', 'v2', http_auth)
        goals = drive_service.files().get_media(fileId=goals_id).execute()

    return render_template('goals.html', goals=json.loads(goals)['goals'])


import math
from datetime import datetime
from collections import Counter
from django.db.models import Q
from recommendation.models import MatchSchedule

def identify_like_teams(user_watch, until=None):
    ''' user_watch is list of matches' id '''
    if until is None:
        until = datetime.now()

    # get how many matches each team already played
    all_matches = MatchSchedule.objects.filter(Q(ms_league=1) & Q(ms_time__lt=until))
    teams_played_counter = Counter()

    for match in all_matches:
        teams_played_counter.update((match.ms_team1.t_id,))
        teams_played_counter.update((match.ms_team2.t_id,))

    # get how many matches each team played in user watch
    user_watch_team_count = Counter()

    for match_id in user_watch:
        match = MatchSchedule.objects.get(ms_id=match_id)
        user_watch_team_count.update((match.ms_team1.t_id,))
        user_watch_team_count.update((match.ms_team2.t_id,))

    # find teams' id that user like
    like_teams = []

    for team_id, count in teams_played_counter.items():
        if user_watch_team_count[team_id] >= math.floor(count * 0.8):
            like_teams.append(team_id)

    return like_teams

def identify_teams_id_with_matches_id(user_watch, until=None):
    ''' user_watch is list of matches' id\n
        return dictionary where key is team's id and value is a list of matches' id\n
        that team already played '''
    teams_matches = {}

    for match_id in user_watch:
        match = MatchSchedule.objects.get(ms_id=match_id)

        if match.ms_team1.t_id in teams_matches:
            teams_matches[match.ms_team1.t_id].append(match_id)
        else:
            teams_matches[match.ms_team1.t_id] = []
            teams_matches[match.ms_team1.t_id].append(match_id)

        if match.ms_team2.t_id in teams_matches:
            teams_matches[match.ms_team2.t_id].append(match_id)
        else:
            teams_matches[match.ms_team2.t_id] = []
            teams_matches[match.ms_team2.t_id].append(match_id)

    return teams_matches

def calculate_jaccard(base_user_watch, compared_user_watch, like_teams, until=None):
    ''' base_user_watch and compared_user_watch are list of matches' id \n
        like_teams is list of teams' id '''
    if until is None:
        until = datetime.now()

    base_user_teams_played = identify_teams_id_with_matches_id(base_user_watch, until=until)
    compared_user_teams_played = identify_teams_id_with_matches_id(compared_user_watch, until=until)

    teams_jaccard_values = {}

    for like_team_id in like_teams:
        base_set = set(base_user_teams_played[like_team_id])
        compared_set = set(compared_user_teams_played.get(like_team_id, list()))

        jacc_val = len(base_set.intersection(compared_set)) / len(base_set.union(compared_set))
        teams_jaccard_values[like_team_id] = jacc_val

    return teams_jaccard_values