import random
import os
import asyncio
import discord
from discord.ext import commands, tasks

client = commands.Bot(command_prefix = '>')

@client.event
async def on_ready():
    print('Logged in as')
    print(client.user.name)
    print(client.user.id)
    print('------')

    #There currently isn't a way to set custom bot status', so for now we will use listening
    display = discord.Activity(name = "for '>' commands", type = 3)
    await client.change_presence(activity = display)

@client.command(description = "Gives the current ping of FERDA BOT - hosted on heroku")
async def ping(ctx):
    """Displays ping of BOT """
    await ctx.send(f'Pong! {round(client.latency * 1000)}ms')

@client.command(aliases = ["cl"], description = "amt - the amount of messages you would like to clear (Don't need to account for the actual command)")
async def clear(ctx, amt):
    """Removes messages from channel"""
    await ctx.channel.purge(limit = int(amt) + 1)

@client.command(description = "name - discord @ of who you'd like to recognize for being FERDA\nreason - reason why they're FERDA")
async def ferda(ctx, name, reason):
    """Recognize one of the boys for being FERDA"""
    await ctx.send(f'{name} is so fucking FERDA')

@client.command(description = "name - discord @ of who you'd like to recognize for being FERDA\nreason - reason why they're not FERDA")
async def negferda(ctx, name, reason):
    """Use this to be toxic and take away FERDA points"""
    await ctx.author.send(":pinching_hand: :eggplant:")

TOKEN= str(os.environ.get("TOKEN"))
client.run(TOKEN)


import socket

from django.conf import settings
from django.utils import timezone

from website.choices import PROCESSED
from website.models import TransformedProject, TransformSelection

class TransformRunner:
    def __init__(self, transformed_project, transform, dry_run, verbosity):
        self.transformed_project = transformed_project
        self.transform = transform
        self.dry_run = dry_run
        self.verbosity = verbosity

        self.host = socket.gethostname()

        self.repo_path = getattr(settings, 'REPO_PATH')
        self.transformed_path = getattr(settings, 'TRANSFORMED_PATH')

    def done(self):
        if not self.dry_run and not self.all_remaining_projects().exists():
            self.transformed_project.status = PROCESSED
            self.transformed_project.datetime_processed = timezone.now()
            self.transformed_project.save()

    def all_projects(self):
        return self.transformed_project.input_projects()

    def projects(self):
        return self.all_projects().filter(host=self.host)

    def all_finished_projects(self):
        if self.transformed_project.src_selector:
            return self.transformed_project.transformed_projects.select_related('src_project')
        return self.transformed_project.transformed_projects.select_related('src_transformer')

    def finished_projects(self):
        return self.all_finished_projects().filter(host=self.host)

    def all_remaining_projects(self):
        if self.transformed_project.src_selector:
            return self.all_projects().exclude(pk__in=self.all_finished_projects().values_list('src_project__pk', flat=True))
        return self.all_projects().exclude(pk__in=self.all_finished_projects().values_list('src_transform__pk', flat=True))

    def remaining_projects(self):
        if self.transformed_project.src_selector:
            return self.projects().exclude(pk__in=self.finished_projects().values_list('src_project__pk', flat=True))
        return self.projects().exclude(pk__in=self.finished_projects().values_list('src_transform__pk', flat=True))

    def debug(self):
        self.run()

    def run(self):
        raise NotImplementedError('transform runners must override the run() method')

    def finish_project(self, proj, istransformed, path = None):
        if self.verbosity >= 3:
            if not path:
                print("project transform failed: " + str(proj))
            else:
                print("saving project transform for: " + str(proj))
        if self.dry_run:
            return

        if istransformed:
            p, _ = TransformedProject.objects.get_or_create(transform=self.transform, src_transform=proj)
        else:
            p, _ = TransformedProject.objects.get_or_create(transform=self.transform, src_project=proj)

        p.host = self.host
        p.path = path
        p.datetime_processed = timezone.now()
        p.save()

        TransformSelection.objects.get_or_create(transformer=self.transformed_project,transformed_project=p)


#!/usr/bin/env python2
# -*- coding: UTF-8 -*-

from pattern.es import parse as parse_es
from pattern.en import parse as parse_en
import pandas as pd
from StringIO import StringIO
import argparse
import re

def df_pos(text, language='es'):

    parsers = dict(es=parse_es,en=parse_en)
    text = unicode(re.sub(u"[\u0100-\uFFFF]", "", text, flags=re.UNICODE), encoding='latin-1').encode('latin-1')

    if text:
        return pd.read_table(
            StringIO("\n".join(parsers[language](text).encode('latin-1', 'ignore').split())),
            sep='/', header=None, quoting=3, encoding='latin-1')

    return pd.DataFrame([[]])

def pattern_pos_filter_text(text, types, language):
    df = df_pos(text, language)
    if not df.empty:
        df = df[[0,1]][~df[1].isin(types)]
    return df.values

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('text', type=str, help="Text to process")
    parser.add_argument('--types', nargs='+', help="List of types to filter")
    parser.add_argument('--language', type=str, default='es', help="Language for POS tagging")
    args = parser.parse_args()

    res = pattern_pos_filter_text(
        args.text,
        args.types,
        args.language
    ).tolist()

    res = [r for r in [r for r in res if len(r) > 1] if isinstance(r[0], str) or isinstance(r[0], unicode)]

    for row in res:
        print_row = '\t'.join(row)
        print(print_row.encode('latin-1'))


from flask import (render_template, url_for, flash,
                   redirect, request, abort, Blueprint)
from flask_login import current_user, login_required
from approot import db
from approot.models import Expense
from approot.expenses.forms import ExpenseForm

expenses = Blueprint('expenses', __name__)

@expenses.route("/expense")
@login_required
def expense():
    page = request.args.get('page', 1, type=int)
    expenses = Expense.query.order_by(Expense.expense_date.desc()).paginate(page=page, per_page=5)
    form = ExpenseForm()
        #return redirect(url_for('expenses.expense'))
    return render_template('expense/expense.html', expenses=expenses, form=form)

@expenses.route("/expense/new", methods=['GET', 'POST'])
@login_required
def new_expense():
    page = request.args.get('page', 1, type=int)
    expenses_from_query = Expense.query.order_by(Expense.expense_date.desc()).paginate(page=page, per_page=5)
    form = ExpenseForm()
    if form.validate_on_submit():
        expense = Expense(description=form.description.data, expense_date=form.expense_date.data,
                        amount=form.amount.data,vat_amount=form.vat_amount.data,Transferrable=form.Transferrable.data, author=current_user)
        db.session.add(expense)
        db.session.commit()
        flash('Your expense has been created!', 'success')
        return redirect(url_for('expenses.expense'))
    #return render_template('expense/create_expense.html', title='New Expense',
    #                       form=form, legend='New Expense')
    return render_template('expense/expense.html', expenses=expenses_from_query, form=form)

@expenses.route("/expense/update/<int:expense_id>", methods=['GET', 'POST'])
@login_required
def update_expense(expense_id):
    page = request.args.get('page', 1, type=int)
    expenses_from_query = Expense.query.order_by(Expense.expense_date.desc()).paginate(page=page, per_page=5)
    expense = Expense.query.get_or_404(expense_id)
    if expense.author != current_user:
        abort(403)
    form = ExpenseForm()
    if form.validate_on_submit():
        expense.description = form.description.data
        expense.amount = form.amount.data
        expense.expense_date = form.expense_date.data
        expense.vat_amount = form.vat_amount.data
        expense.Transferrable = form.Transferrable.data
        db.session.commit()
        flash('Your expense has been updated!', 'success')
        return redirect(url_for('expenses.expense'))
        """ Commented to see if this will have any negative effect on the flow - I think it won't
    elif request.method == 'GET':
        form.description.data = expense.description
        form.amount.data = expense.amount
        form.expense_date.data = expense.expense_date
        form.vat_amount.data = expense.vat_amount
        form.Transferrable.data = expense.Transferrable """
    #return render_template('expense/expense.html', title='Update Expense',
    #                       form=form, legend='Update Expense')
    return render_template('expense/expense.html', expenses=expenses_from_query, form=form, expense_id=expense_id)

@expenses.route("/expense/delete/<int:expense_id>", methods=['POST'])
@login_required
def delete_expense(expense_id):
    expense = Expense.query.get_or_404(expense_id)
    if expense.author != current_user:
        abort(403)
    db.session.delete(expense)
    db.session.commit()
    flash('Your expense has been deleted!', 'success')
    return redirect(url_for('expenses.expense'))


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Reasoning module.

This module is the core of the system. It interacts with the knowledge base
via a prolog interface defined in a second module in this package to reason
over it.
It can be activated via the GUI or via the senses, i.e., the sensors installed
on the robot.

"""

import rospy

from grounding_action_server import GroundingActionServer
from errors import *
from prolog_interface import PrologInterface
from rospy_helper import *

class Reasonig(object):

    def __init__(self, pl_interface):
        """ Initializer """
        self.interlocutor = ""
        self.pl_interface = pl_interface

    def callback(self, data):
        """ speech recognition response callback
        @param data: recognized result. 2 attributes : action (e.g. GoTo.) |  target (e.g. Kitchen.)
        """
        # --- extract informations ---
        target = data.target
        action = data.action
        try:
            self.pl_interface.check_actions(action)
            self.pl_interface.check_targets(target)
        except (FailedToUnderstandTarget, FailedToUnderstandAction) as e :
            speak(e.message)
            update_internal_state(e.message)
        else:
            if (action == "Go"):
                #send_command(target)
                msg = "I will!"
                speak(msg)
            else:
                msg = "I am sorry. This capability is not available yet."
                speak(msg)


import os, pickle, copy
# import cv2
import numpy as np
from PIL import Image
from utils.data_aug import clamp_bboxs, convert_bboxs
from args import train_transforms, val_transforms

def parse_lines(lines):
    '''
    Given lines from train/val annotation file, return img_paths, bboxs and labels of every img.
    line format: img_path, xmin, ymin, xmax, ymax, label, ... (bboxs in the format of [xmin, ymin, xmax, ymax, label])
    return :
        img_paths : string, path of each img
        bboxs : list, elements in the list are in the shape of [N, 4], N is bbox nums of each img,
            4 stands for [xmin, ymin, xmax, ymax] respectively
        labels : list, elements in the the are in the shape of [N]
        bboxs_num : Numpy.array object, bboxs num of each image
    '''
    idxs, img_paths, bboxs, labels, bboxs_num = [], [], [], [], []
    for line in lines:
        if isinstance(line, str) is False:
            try:
                line = line.decode('utf-8')
            except:
                raise Exception('Error accur when parse the annotation. There may be some special characters that are not included in utf-8 in your annotation file')
        s = line.strip().split(' ')
        idxs.append(int(s.pop(0)))
        img_paths.append(s.pop(0))
        if( len(s) == 0):
            idxs.pop(-1)
            img_paths.pop(-1)
            continue
        assert len(s) % 5 == 0,  'Annotation Error. Please check your annotation files. Make sure the completeness of bboxs and ensure the img paths and labels in the file'
        num_bbox = len(s) // 5
        bbox, label = [], []
        for i in range(num_bbox):
            bbox.append([float(s[ i * 5 ]), float(s[ i * 5 + 1 ]), float(s[ i * 5 + 2 ]), float(s[ i * 5 + 3 ])])      #xmin, ymin, xmax, ymax, respectively
            label.append(int(s[ i * 5 + 4 ]))
        if(len(bbox) > 35):
            bbox = bbox[:35]
            label = label[:35]
        bboxs.append(np.array(bbox, np.float32))
        labels.append(np.array(label, np.int64))
        bboxs_num.append(len(bbox))

    return np.array(idxs), img_paths, bboxs, labels, np.array(bboxs_num)

def imgs_loader(img_paths, root=None):
    '''
    Given a list of img_paths (or a single img path), return the list of imgs (PIL.Image object) and their sizes
    Note : If your model need images with BGR format, please uncomment the line : imgs[-1] = Image.fromarray(cv2.cvtColor(np.asarray(imgs[-1]),cv2.COLOR_RGB2BGR))
    return :
        imgs : list, PIL.Image object
        img_size : list, img sizes
    '''
    root = '' if root is None else root
    if isinstance(img_paths, list) is False:
        img_paths = [img_paths]
    imgs, img_size = [], []
    for img_path in img_paths:
        imgs.append(Image.open(os.path.join(root, img_path)).convert('RGB'))
#         imgs[-1] = Image.fromarray(cv2.cvtColor(np.asarray(imgs[-1]),cv2.COLOR_RGB2BGR))
        img_size.append(list(imgs[-1].size))
    return imgs, img_size

#return images, bboxs, labels, img_size, img_idxs
def get_batch_data(lines, bbox_mode, mode='train'):
    '''
    Function for getting batch data, receive annotation lines and return images, boxes, labels, labels and idxs
    '''
    bbox_mode = bbox_mode.decode()

    idxs, img_paths, bboxs, labels, bboxs_num = parse_lines(lines)
    imgs, img_size = imgs_loader(img_paths)
    if bbox_mode == 'xywh':                                          #if the bboxs is xywh format, then change it to xyxy format
        bboxs = [convert_bboxs(box, 'xyxy') for box in bboxs]
    bboxs = clamp_bboxs(bboxs, img_size, remove_empty=True)

    transform = train_transforms if mode == 'train' else val_transforms
    for transform in train_transforms:
        imgs, bboxs = transform(imgs, bboxs)
    imgs, batch_img_size = imgs
    bboxs, bboxs_num = bboxs
    batch_labels = np.ones((bboxs.shape[:2]), dtype=np.float32) * -1
    for batch_label, label in zip(batch_labels, labels):
        batch_label[:len(label)] = label
    return imgs, bboxs, batch_labels, batch_img_size.astype(np.int64), \
        np.array(img_size, dtype=np.int64), bboxs_num.astype(np.int64), idxs.astype(np.int64)


__author__ = 'bfy'

import logging
import datetime

logger = logging.getLogger("baiduSpider")
logger.setLevel(logging.DEBUG)

now = datetime.datetime.now()
now = str(now).split(' ')[0]
fileHandler = logging.FileHandler(r'log/' + now)
fileHandler.setLevel(logging.DEBUG)

consoleHandler = logging.StreamHandler()
consoleHandler.setLevel(logging.DEBUG)

formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fileHandler.setFormatter(formatter)
consoleHandler.setFormatter(formatter)

logger.addHandler(fileHandler)
logger.addHandler(consoleHandler)


import torch
import torchvision
import numpy as np

def compute_point(b,w,point):      # loss function before ml
	tot=0
	for i in range(0,len(point)):
		x=point[i,0]
		y=point[i,1]
		tot+=(y-(w*x+b))**2
	return tot/float(len(point))

def step_g(b_c,w_c,point,learningrate):     #single step_gradient
	b_g=0
	w_g=0
	N=float(len(point))
	for i in range(0,len(point)):
		x=point[i,0]
		y=point[i,1]
		b_g+= -(2/N)*(y-((w_c*x)+b_c))
		w_g+= -(2/N)*x*(y-((w_c*x)+b_c))
	new_b=b_c-(learningrate*b_g)
	new_m=w_c-(learningrate*w_g)
	return [new_b,new_m]

def gradient(point,start_b,start_m,learn_rate,num_iter):     #loop step gradient
	b=start_b
	m=start_m
	for i in range(num_iter):
		b,m=step_g(b,m,point,learn_rate)
		print("b={0},m={1}".format(b,m))
	return [b,m]

def main():
	point=np.genfromtxt("data.csv",delimiter=",")
	learn_rate=0.0001
	inib=0
	inim=0
	num_iter=100000
	print("Starting gradient descent at b = {0}, m = {1}, error = {2}"
		  .format(inib,inim,compute_point(inib,inim,point)))
	print("Running>>>>>")
	[b,m]=gradient(point,inib,inim,learn_rate,num_iter)
	print("After {0} iterations b={1} ,m={2} ,error={3}".
		format(num_iter,b,m,compute_point(b,m,point)))

main()


from django.conf.urls import include, url, patterns
from django.contrib.sitemaps.views import sitemap
from django.contrib.sitemaps import FlatPageSitemap, GenericSitemap, Sitemap
from django.core.urlresolvers import reverse
from vmaig_blog.settings import MEDIA_ROOT

from django.contrib import admin

from blog.models import Article, News, Category, Column

class StaticViewSitemap(Sitemap):
    priority = 1.0
    changefreq = 'daily'

    def items(self):
        return ['index-view', 'news-view']

    def location(self, item):
        return reverse(item)

sitemaps = {
    'article-is-top': GenericSitemap(
                {
                    'queryset': Article.objects.filter(
                                    status=0, is_top=True
                                ).all(),
                    'date_field': 'pub_time'
                },
                priority=1.0,
                changefreq='daily'
            ),
    'article-is-not-top': GenericSitemap(
                {
                    'queryset': Article.objects.filter(status=0).all(),
                    'date_field': 'pub_time'
                },
                priority=0.8,
                changefreq='daily'
            ),
    'news': GenericSitemap(
                {
                    'queryset': News.objects.all(),
                    'data_field': 'pub_time'
                },
                priority=0.6,
                changefreq='daily'
            ),
    'category': GenericSitemap(
                {
                    'queryset': Category.objects.all()
                },
                priority=0.9,
                changefreq='daily'
            ),
    'column': GenericSitemap(
                {
                    'queryset': Column.objects.all()
                },
                priority=0.9,
                changefreq='daily'
            ),
    'static': StaticViewSitemap
}

urlpatterns = [
    url(r'^admin/', include(admin.site.urls)),
    url(r'^ueditor/',include('DjangoUeditor.urls')),
    url(r'', include('blog.urls')),
    url(r'', include('vmaig_comments.urls')),
    url(r'', include('vmaig_auth.urls')),
    url(r'^sitemap\.xml$', sitemap, {'sitemaps': sitemaps},
        name='django.contrib.sitemaps.views.sitemap'),
    url(r'^media/(?P<path>.*)$', 'django.views.static.serve', {'document_root': MEDIA_ROOT}),
]

urlpatterns += patterns('api.views',
    url(r'^api/$', 'index'),
    url(r'^api/docs/$', 'docs'),
    url(r'^api/docs/(?P<json>\w+)/$', 'module'),
)


import os

a = os.getcwd()

file_name = a+"my_task.txt"

with open(file_name,'w') as f:
    f.write("1. Attend the meetup")
    f.write("2. Go home and get sleep")
    f.write("3. Make sure you had lunch before going to bed.")
    f.close()
    print(f.seek(0))

print("File written successfully")

##append
##add
##see


import csv
import RA
import pandas as pd

def _get_col_datatypes(fin):
    """modified from https://stackoverflow.com/questions/2887878/importing-a-csv-file-into-a-sqlite3-database-table-using-python """
    dr = csv.DictReader(fin) # comma is default delimiter
    fieldTypes = {}
    dtypes = []
    for entry in dr:
        feildslLeft = [f for f in dr.fieldnames if f not in fieldTypes.keys()]
        if not feildslLeft: break # We're done
        for field in feildslLeft:
            data = entry[field]

            # Need data to decide
            if len(data) == 0:
                continue
            sqltype = get_types(data)
            fieldTypes[field] = sqltype

    if len(feildslLeft) > 0:
        raise Exception("Failed to find all the columns data types - Maybe some are empty?")

    return fieldTypes

def get_types(data):
    if data.isdigit(): # check integer
        return "REAL"
    else: # default to text
        try: # try real
            temp = float(data)
        except:
            return "TEXT"
        else:
            return "REAL"
    return "UNKNOWN"

def escapingGenerator(f):
    """from https://stackoverflow.com/questions/2887878/importing-a-csv-file-into-a-sqlite3-database-table-using-python """
    for line in f:
        yield line.encode("ascii", "xmlcharrefreplace").decode("ascii")

def csvToDb(csvFile, con=None, table_name=None):
    from lazypandas import lazyDf
    if table_name is None:
        print('no table name found')
        table_name = 'ads'
    else:
        print('table name', table_name)

    if con is None:
        print('No con. Creating engine')
        con = sqlite3.connect(":memory:")
    with open(csvFile, mode='r') as fin: # necessary ?
        dt = _get_col_datatypes(fin)
        fin.seek(0)
        reader = csv.DictReader(fin)
        # Keep the order of the columns name just as in the CSV
        fields = reader.fieldnames
        cols = []
        # Set field and type
        for f in fields:
            cols.append("%s %s" % (f, dt[f]))
    df = pd.read_csv(csvFile, engine='c' ) #  dtype=dtypes
    managed_to_sql(df, table_name, con=con, index_label='auto_index')
    dotted_fields = {}
    dotted_datatype = {}
    dotted_fields[table_name] = fields
    dotted_fields[table_name].append('auto_index')
    dt['auto_index'] = "INTEGER"
    dotted_datatype = dt # [table_name]
    ret = lazyDf([table_name], dotted_fields, dotted_datatype, RA.RA_from(table_name, fields))
    ret.add_engine(con)
    return ret

def out_to_pd(out):
    colname = [d[0] for d in out.description]
    return pd.DataFrame(data=out.fetchall(), columns=colname)

def column_equal(cols, attr):
    ret = ""
    print(cols)
    for table in cols:
        for col in cols[table][:-1]:
            ret += ' ' + col + ' = ' + str(attr) + ' and '
        ret += ' ' + cols[table][-1] + ' = ' + str(attr) + ' '
    return ret

def managed_to_sql(df, table_name, con, index_label=None):
    tables_already_present_with_stupid_tuple = \
        con.execute("select name from sqlite_master where type='table';").fetchall()
    tables_already_present = [x[0] for x in tables_already_present_with_stupid_tuple]
    if table_name in tables_already_present:
        print('WARNING::Overwriting table_name '+table_name+' inside db', flush=True)
        con.execute('drop table ' + table_name + ' ;')
    df.to_sql(table_name, con=con, index_label=index_label)

def update_sql_data(table, attribute, index, value, con):
    N = 1024
    L = len(index)
    if isinstance(value, (str, int, float)):
        cmd = \
        "update " + str(table) + \
        " set " + str(attribute) + "=" + str(value) + \
        " where " + table + ".auto_index=" + "(?)" + ";"
        con.executemany(cmd, str(index))
    else:
        for bl_idx, _ in enumerate(index[::N]):
            cmd = ""
            for idx, v in zip(index[bl_idx:min(L, bl_idx+N)], value[bl_idx:min(L, bl_idx+N)]):
                cmd = \
                    "update " + str(table) + \
                    " set " + str(attribute) + "=" + str(v) + \
                    " where " + table + ".auto_index=" + str(idx) + ";"
            con.executescript(cmd)


import unittest
import deck

class DeckTests(unittest.TestCase):

    def setUp(self):
        self.deck1 = deck.Deck()
        self.deck1.prepare(1)

        self.deck2 = deck.Deck()
        self.deck2.prepare(2)

        self.deck3 = deck.Deck()
        self.deck3.prepare(3)

        self.empty_deck = deck.Deck()

    def test_card_count(self):
        card_count = 0

        while self.deck1.has_cards():
            self.deck1.next_card()
            card_count += 1

        self.assertTrue(card_count == 54)

    def test_empty_deck(self):
        self.assertFalse(self.empty_deck.has_cards())

if __name__ == '__main__':
    unittest.main()


import sys
input = sys.stdin.readline

n, k = map(int, input().split())
visited = [False] * 100001
ans = 0

# [0]-> val, [1] -> time
q = []
q.append([n, 0])
visited[n] = True
while q:
    now = q.pop(0)
    if now[0] == k:
        ans = now[1]
        break
    if now[0]-1 >= 0 and not visited[now[0]-1]:
        q.append([now[0]-1, now[1]+1])
        visited[now[0]-1] = True
    if now[0]+1 <= 100000 and not visited[now[0]+1]:
        q.append([now[0]+1, now[1]+1])
        visited[now[0]+1] = True
    if now[0]*2 <= 100000 and not visited[now[0]*2]:
        q.append([now[0]*2, now[1]+1])
        visited[now[0]*2] = True

print(ans)


import json

from deepdiff import DeepDiff

class JSONUtils:
    """JSONUtils is common resource for simple json helper keywords."""

    def json_equals(self, left, right):
        """JSON Equals takes in two strings or json objects, converts them into json if needed and then compares them, returning if they are equal or not."""
        if isinstance(left, basestring):
            left_json = json.loads(left);
        else:
            left_json = left;
        if isinstance(right, basestring):
            right_json = json.loads(right);
        else:
            right_json = right;

        ddiff = DeepDiff(left_json, right_json, ignore_order=True);
        if ddiff == {}:
            return True;
        else:
            return False;

    def make_list_into_dict(self, listOfDicts, key):
        """ Converts a list of dicts that contains a field that has a unique key into a dict of dicts """
        d = {}
        if isinstance(listOfDicts, list):
            for thisDict in listOfDicts:
                v = thisDict[key]
                d[v] = thisDict
        return d

    def find_element_in_array(self, searchedArray, key, value):
        """ Takes in an array and a key value, it will return the items in the array that has a key and value that matches what you pass in """
        elements = [];
        for item in searchedArray:
            if key in item:
                if item[key] == value:
                    elements.append(item);
        return elements;


#2020-11-18 (8일차)
#model_load()
#fit 이전에 save: 모델만

import matplotlib.pyplot as plt
import numpy as np
from tensorflow.keras.datasets import mnist #텐서플로우에서 제공해 준다(수치로 변환해서 제공)

#train_test_split 할 필요 없이 알아서 나눠 준다
(x_train, y_train), (x_test, y_test) = mnist.load_data() #괄호 주의

#60000장 * 28pixel * 28pixel
# print(x_train.shape, x_test.shape) #(60000, 28, 28)(10000, 28, 28)
# print(y_train.shape, y_test.shape) #(60000, )      (10000,)        : 스칼라

# print(x_train[0])
# print(y_train[1]) #label

# plt.imshow(x_train[0], 'gray')
# plt.show()

#8은 2보다 4배의 가치? 3은 1보다 3배의 가치? no
#One-Hot Encoder
#y_train: 60000, -> OneHotEncoding : 1 0 0 0 0 0 0 0 0 0 (60000, 10) : 분류가 10개니까 (0~9)

#1. 데이터 전처리: OneHotEncoding 대상은 Y
from tensorflow.keras.utils import to_categorical
y_train = to_categorical(y_train)
y_test = to_categorical(y_test)
print(y_train)

# print(y_train.shape, y_test.shape)
# print(y_train[0]) #y_train[0]=5 -> [0. 0. 0. 0. 0. 1. 0. 0. 0. 0.]

#shape 바꿀 줄 알아야 함
#60000, 14, 14, 4도 가능하고 60000, 28, 14, 2도 가능
#LSTM으로도 바꿀 수 있다

x_train = x_train.reshape(60000, 28, 28, 1).astype('float32')/255.
x_test = x_test.reshape(10000, 28, 28, 1).astype('float32')/255.
                        #x_test.shape[0], x_test.shape[1] ...

#predict data, answer data
x_predict = x_train[20:30]
y_answer = y_train[20:30]

#CNN에 넣을 수 있는 4차원 reshape + y도 onehotencoding
#scaler 사용해야: 어떤 게 더 좋을지는 해 봐야 안다
#지금 이 상황에서 M은 255라는 걸 알고 있음. 그러므로 MinMax에서는 255로 나누면 0~1 사이로 수렴 가능

# print(x_train[0])

#2. 모델
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Conv2D, MaxPooling2D, Flatten

# model = Sequential()
# model.add(Conv2D(30, (2, 2), padding='same', input_shape=(28, 28, 1))) #padding 주의!
# model.add(Conv2D(50, (2, 2), padding='valid'))
# model.add(Conv2D(120, (3, 3))) #padding default=valid
# model.add(Conv2D(200, (2, 2), strides=2))
# model.add(Conv2D(30, (2, 2)))
# model.add(MaxPooling2D(pool_size=2)) #pool_size default=2
# model.add(Flatten())
# model.add(Dense(10, activation='relu')) #flatten하면서 곱하고 dense에서 또 100 곱함
#                                         #Conv2d의 activation default='relu'
#                                         #LSTM의 activation default='tanh'
# #MaxPooling2D-Flatten:reshape 개념

# model.add(Dense(10, activation='softmax')) #softmax** : 2 이상 분류(다중분류)의 activation은 softmax, 2진분류는 sigmoid
#                                             #즉 softmax를 사용하려면 OneHotEncoding 해야

#모델 1 (fit 이전에 저장) 로드
from tensorflow.keras.models import load_model
model = load_model('./save/model_test01_1.h5')
model.summary()

#3. 컴파일, 훈련

modelpath = './model/mnist-{epoch:02d}-{val_loss:.4f}.hdf5'

#root 폴더(=Study) 밑의 model 폴더에
#.hd5f라는 확장자 => *.h5 (모델 확장자와 거의 동일하다)
#epoch:두자릿수 정수 표기-val_loss:소수 넷째자리까지 표기 -> 이 파일명으로 생성 : 파일명만 봐도 대략 어느 지점이 좋은지 알 수 있음

from tensorflow.keras.callbacks import EarlyStopping, TensorBoard, ModelCheckpoint

early_stopping = EarlyStopping(monitor='val_loss', patience=10, mode='auto')

#val_loss가 개선이 생길 때마다 history를 저장하고 싶다!
#정의
cp = ModelCheckpoint(filepath=modelpath,
                     monitor='val_loss',
                     save_best_only=True,
                     mode='auto'
) #계속 저장될 것 (계속 좋아질 테니까) 그러다가 튀는 구간 있으면 또 저장 안 되고... 다시 뚫고 내려가는 거 저장되고...
  #좋은 시점 저장됨
#model 폴더 잘 보기!
#04-어쩌고 후에 07-어쩌고가 생겼다면 5, 6은 별로였다는 뜻

#log가 들어갈 폴더='graph'
#여기까지 해서 graph 폴더 생기고 자료들 들어가 있으면 텐서보드 쓸 준비 ok
#단, 로그가 많으면 겹쳐서 보일 수 있으니 그럴 땐 로그 삭제하고
to_hist = TensorBoard(log_dir='graph', histogram_freq=0,
                      write_graph=True, write_images=True
)

model.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy']) #"mean_squared_error" (풀네임도 가능하다)

#model.fit()은 history 반환
hist = model.fit(x_train, y_train, epochs=100, batch_size=32, verbose=1,
          validation_split=0.2, callbacks=[early_stopping, cp]) #tensorboard 쓰고 싶으면 여기다가 to_hist도...

#fit에 있는 네 가지
loss = hist.history['loss']
val_loss = hist.history['val_loss']
acc = hist.history['accuracy']
val_acc = hist.history['val_accuracy']

#4. 평가, 예측

#얘네는 evaluate에서 나온 애들

result = model.evaluate(x_test, y_test, batch_size=32)
print("loss: ", result[0])
print("acc: ", result[1])

#시각화
#plot에는 x, y가 들어간다 (그래야 그래프가 그려짐)
import matplotlib.pyplot as plt

plt.figure(figsize=(10, 6))
#단위 뭔지 찾아볼 것!!!
#pyplot.figure 는 매개 변수에 주어진 속성으로 새로운 도형을 생성합니다.
#figsize 는 도형 크기를 인치 단위로 정의합니다.

plt.subplot(2, 1, 1) #2, 1, 1 -> 두 장 중의 첫 번째의 첫 번째 (2행 1열에서 첫 번째)
# plt.plot(hist.history['loss'],) #loss값이 순서대로 감
plt.plot(hist.history['loss'], marker='.', c='red', label='loss') #loss값이 순서대로 감, 마커는 점, 색깔은 빨간색, 라벨은 loss
plt.plot(hist.history['val_loss'], marker='.', c='blue', label='val_loss')

plt.grid() #모눈종이 배경
plt.title('loss')
plt.ylabel('loss')
plt.xlabel('epoch')

#위에서 라벨 명시 후 위치 명시
#그림의 위치(location)는 상단: label:loss, label:val_loss 이 둘이 박스로 해서 저 위치에 나올 것
plt.legend(loc='upper right')

plt.subplot(2, 1, 2) #2, 1, 1 -> 2행 1열 중 두 번째 (두 번째 그림)
# plt.plot(hist.history['loss'],) #loss값이 순서대로 감
plt.plot(hist.history['accuracy'], marker='.', c='red') #loss값이 순서대로 감, 마커는 점, 색깔은 빨간색, 라벨은 loss
plt.plot(hist.history['val_accuracy'], marker='.', c='blue')

plt.grid() #모눈종이 배경
plt.title('accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')

#여긴 라벨만 명시
plt.legend(['accuracy', 'val_accuracy'])

#보여 줘
plt.show()

#정답
y_answer = np.argmax(y_answer, axis=1)

#예측값
y_predict = model.predict(x_predict)
y_predict = np.argmax(y_predict, axis=1)

print("예측값: ", y_predict)
print("정답: ", y_answer)

'''
loss:  0.09025763720273972
acc:  0.9763000011444092
'''


import os
import time
import unittest

from vika import Vika

from . import TEST_API_BASE, TEST_API_TOKEN, TEST_TABLE

class TestUploadFile(unittest.TestCase):
    def setUp(self):
        self.vika = Vika(TEST_API_TOKEN)
        self.vika.set_api_base(TEST_API_BASE)
        self.dst = self.vika.datasheet(TEST_TABLE)

    def test_upload_file(self):
        test_url = "https://img9.doubanio.com/view/subject/s/public/s26849345.jpg"
        test_file = self.dst.upload_attachment(test_url)
        self.assertIsNotNone(test_file.get("token"))

        time.sleep(2)

        test_local_file = "vikaji.png"
        filepath = os.path.join(os.path.dirname(__file__), test_local_file)
        test_file = self.dst.upload_file(filepath)
        self.assertEqual(test_file.get("mimeType"), "image/png")

if __name__ == "__main__":
    unittest.main()


# Script name: app2.py

# call the function from the file of mathFunction2.py

from mathFunction2 import add, mul

add1 = add(5, 35)

mul1 = mul(3336.56, 5.266)

print(add1)
print(mul1)

# Output
# 40
# 15


import torch.nn as nn
import torch
from collections import OrderedDict

class RagaDetector(nn.Module):
    def __init__(self):
        super(RagaDetector, self).__init__()

        self.norm0 = nn.BatchNorm2d(1)
        self.encoder = nn.Sequential(OrderedDict([
            ('norm0', nn.BatchNorm2d(1)),

            ('conv1', nn.Conv2d(1, 64, 3, padding=1)),
            ('norm1', nn.BatchNorm2d(64)),
            ('elu1', nn.ELU()),
            ('pool1', nn.MaxPool2d(2, 2)),
            ('drop1', nn.Dropout(p=0.1)),

            ('conv2', nn.Conv2d(64, 128, 3, padding=1)),
            ('norm2', nn.BatchNorm2d(128)),
            ('elu2', nn.ELU()),
            ('pool2', nn.MaxPool2d(3, 3)),
            ('drop2', nn.Dropout(p=0.1)),

            ('conv3', nn.Conv2d(128, 128, 3, padding=1)),
            ('norm3', nn.BatchNorm2d(128)),
            ('elu3', nn.ELU()),
            ('pool3', nn.MaxPool2d(4, 4)),
            ('drop3', nn.Dropout(p=0.1)),

            ('conv4', nn.Conv2d(128, 128, 3, padding=1)),
            ('norm4', nn.BatchNorm2d(128)),
            ('elu4', nn.ELU()),
            ('pool4', nn.MaxPool2d(4, 4)),
            ('drop4', nn.Dropout(p=0.1))
        ]))

        self.lstm = nn.LSTM(128, 72, num_layers=2, dropout=0.1)

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(
                    m.weight, mode='fan_out', nonlinearity='relu')
            elif isinstance(m, nn.BatchNorm2d):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)

    def forward(self, x, hidden):
        x = x.transpose(0, 1).unsqueeze(1)
        x = self.encoder(x)
        batch_size = x.shape[0]
        x = x.transpose(0, 2).transpose(1, 2).view(-1, batch_size, 128)
        x, hidden = self.lstm(x, hidden)

        return x, hidden

    def init_hidden(self, bsz):
        weight = next(self.parameters())
        return (weight.new_zeros(2, bsz, 72),
                weight.new_zeros(2, bsz, 72))


#! /usr/bin/env python3

'''
Sherlock: Find Usernames Across Social Networks Module

This module contains the main logic to search for usernames at social
networks.
'''

# ==================== Imports ==================== #
import os
import sys
import json
import csv
import re
from concurrent.futures import ThreadPoolExecutor
import requests
from colorama import Fore, Style, init
from torrequest import TorRequest

from watson import *

# TODO: fix tumblr

# ==================== Main ==================== #
class Sherlock:
    def __init__(self, username, index_by_original_query=False, source=os.path.join(os.path.dirname(os.path.realpath(__file__)), "data.json")):
        '''Create an instance of Sherlock that can be used to look up usernames on social media.

        Keyword Arguments:
        username                    -- The username to look up.
        index_by_original_query     -- Index the results by original query, if false, results are indexed by the actual name of the social media site.
        source                      -- The source of the site data JSON used by Sherlock, can be a URL.
        '''
        self.results = dict()
        self.username = username
        self.indexed_by_original_query = index_by_original_query
        if source is None:
            self.source = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data.json")
        else:
            self.source = source

    def __prepare_site_data(self, sites, ranked=False):
        '''Prepares a site data dictionary of all the sites that Sherlock will analyze.

        Keyword Arguments:
        sites       -- a list of sites as strings
        ranked      -- rank the sites by their alexa ranking

        Return Value:
        Dictionary containing all the site data of the given sites. This site data
        is used by Sherlock to check if the username exists at those sites.
        '''
        response_json_online = None
        # Try to load json from website.
        try:
            response_json_online = requests.get(url=self.source)
        except requests.exceptions.MissingSchema:  # In case the schema is wrong it's because it may not be a website
            pass

        # Check if the response is appropriate.
        if response_json_online is not None and response_json_online.status_code == 200:
            # Since we got data from a website, try to load json and exit if parsing fails.
            try:
                site_data_all = response_json_online.json()
                return site_data_all
            except ValueError:
                print(Style.BRIGHT + Fore.RED + "Invalid JSON from website!")
                Loader.stop_all()
                sys.exit(1)
                pass

        # Load the data
        data_file_path = self.source
        with open(data_file_path, "r", encoding="utf-8") as raw:
            site_data_all = json.load(raw)

        if sites is None:
            # Not desired to look at a sub-set of sites
            site_data = site_data_all
        else:
            # User desires to selectively run queries on a sub-set of the site list.
            # Make sure that the sites are supported & build up pruned site database.
            site_data = dict()
            site_missing = []
            for site in sites:
                for existing_site in site_data_all:
                    if site.lower() in existing_site.lower():
                        site_data[existing_site] = site_data_all[existing_site]
                        site_data[existing_site]['original_query'] = site
                if not site_data:
                    # Build up list of sites not supported for future error message.
                    site_missing.append(f"'{site}'")

            if site_missing:
                print(Style.BRIGHT + Fore.RED + f"Error: Desired sites not found: {', '.join(site_missing)}.")
                Loader.stop_all()
                sys.exit(1)

        if ranked:
            # Sort data by rank
            site_dataCpy = dict(site_data)
            ranked_sites = sorted(site_data, key=lambda k: ("rank" not in k, site_data[k].get("rank", sys.maxsize)))
            site_data = {}
            for site in ranked_sites:
                site_data[site] = site_dataCpy.get(site)

        return site_data

    def check(self, sites, verbose=False, tor=False, unique_tor=False, proxy=None, silent_on_error=False, ranked=False):
        '''Run Sherlock Analysis.

        Checks for existence of username on various social media sites.

        Keyword Arguments:
        site_data              -- Dictionary containing all of the site data.
        verbose                -- Boolean indicating whether to give verbose output.
        tor                    -- Boolean indicating whether to use a tor circuit for the requests.
        unique_tor             -- Boolean indicating whether to use a new tor circuit for each request.
        proxy                  -- String indicating the proxy URL.
        silent_on_error        -- Will not print errors to STDOUT.
        ranked                 -- Rank the results by alexa.com rankings.

        Return Value:
        Dictionary containing results from report.  Key of dictionary is the name
        of the social network site, and the value is another dictionary with
        the following keys:
            url_main:      URL of main site.
            url_user:      URL of user on site (if account exists).
            exists:        String indicating results of test for account existence.
            http_status:   HTTP status code of query which checked for existence on
                        site.
            response_text: Text that came back from request.  May be None if
                        there was an HTTP error when checking for existence.
        '''

        # Get site data
        if isinstance(sites, str):
            sites = [sites]
        site_data = self.__prepare_site_data(sites, ranked)

        # A user agent is needed because some sites don't
        # return the correct information since they think that
        # we are bots
        headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:55.0) Gecko/20100101 Firefox/55.0'
        }

        # Allow 1 thread for each external service, so `len(site_data)` threads total
        executor = ThreadPoolExecutor(max_workers=len(site_data))

        # Create session based on request methodology
        if tor or unique_tor:
            underlying_request = TorRequest()
            underlying_session = underlying_request.session
        else:
            underlying_session = requests.session()
            underlying_request = requests.Request()

        # Create multi-threaded session for all requests. Use our custom FuturesSession that exposes response time
        session = ElapsedFuturesSession(executor=executor, session=underlying_session)

        # Results from analysis of all sites
        results_total = self.results

        # First create futures for all requests. This allows for the requests to run in parallel
        for social_network, net_info in site_data.items():

            # Results from analysis of this specific site
            results_site = {}

            # Record URL of main site
            results_site['url_main'] = net_info.get("urlMain")

            # Don't make request if username is invalid for the site
            regex_check = net_info.get("regexCheck")
            if regex_check and re.search(regex_check, self.username) is None:
                # No need to do the check at the site: this user name is not allowed.
                print((Style.BRIGHT + Fore.WHITE + "[" +
                    Fore.RED + "-" +
                    Fore.WHITE + "]" +
                    Fore.GREEN + " {}:" +
                    Fore.YELLOW + " Illegal Username Format For This Site!").format(social_network))
                print()
                results_site["exists"] = "illegal"
            else:
                # URL of user on site (if it exists)
                url = net_info["url"].format(self.username)
                results_site["url_user"] = url

                request_method = session.get
                if social_network != "GitHub":
                    # If only the status_code is needed don't download the body
                    if net_info["errorType"] == 'status_code':
                        request_method = session.head

                if net_info["errorType"] == "response_url":
                    #Site forwards request to a different URL if username not
                    #found.  Disallow the redirect so we can capture the
                    #http status from the original URL request.
                    allow_redirects = False
                else:
                    #Allow whatever redirect that the site wants to do.
                    #The final result of the request will be what is available.
                    allow_redirects = True

                # This future starts running the request in a new thread, doesn't block the main thread
                if proxy != None:
                    proxies = {"http": proxy, "https": proxy}
                    future = request_method(url=url, headers=headers,
                                            proxies=proxies,
                                            allow_redirects=allow_redirects
                                        )
                else:
                    future = request_method(url=url, headers=headers,
                                            allow_redirects=allow_redirects
                                        )

                # Store future in data for access later
                net_info["request_future"] = future

                # Reset identify for tor (if needed)
                if unique_tor:
                    underlying_request.reset_identity()

            # Add this site's results into final dictionary with all of the other results.
            if self.indexed_by_original_query:
                results_total[site_data[social_network]['original_query']] = results_site
            else:
                results_total[social_network] = results_site

        # Open the file containing account links
        # f = open_file(fname)

        # Core logic: If tor requests, make them here. If multi-threaded requests, wait for responses
        for social_network, net_info in site_data.items():

            # Retrieve results again
            results_site = results_total.get(social_network)

            # Retrieve other site information again
            url = results_site.get("url_user")
            exists = results_site.get("exists")
            if exists is not None:
                # We have already determined the user doesn't exist here
                continue

            # Get the expected error type
            error_type = net_info["errorType"]

            # Default data in case there are any failures in doing a request.
            http_status = "?"
            response_text = ""

            # Retrieve future and ensure it has finished
            future = net_info["request_future"]
            r, error_type, response_time = get_response(request_future=future,
                                                        error_type=error_type,
                                                        social_network=social_network,
                                                        verbose=verbose)

            # Attempt to get request information
            try:
                http_status = r.status_code
            except:
                pass
            try:
                response_text = r.text.encode(r.encoding)
            except:
                pass

            if error_type == "message":
                error = net_info.get("errorMsg")
                # Checks if the error message is in the HTML
                if not error in r.text:
                    # print_found(social_network, url, response_time, verbose)
                    exists = "yes"
                else:
                    # print_not_found(social_network, response_time, verbose)
                    exists = "no"

            elif error_type == "status_code":
                # Checks if the status code of the response is 2XX
                if not (r.status_code >= 300 or r.status_code < 200):
                    # print_found(social_network, url, response_time, verbose)
                    exists = "yes"
                else:
                    # print_not_found(social_network, response_time, verbose)
                    exists = "no"

            elif error_type == "response_url":
                # For this detection method, we have turned off the redirect.
                # So, there is no need to check the response URL: it will always
                # match the request.  Instead, we will ensure that the response
                # code indicates that the request was successful (i.e. no 404, or
                # forward to some odd redirect).
                if (r.status_code >= 200) and (r.status_code < 300):
                    # print_found(social_network, url, response_time, verbose)
                    exists = "yes"
                else:
                    # print_not_found(social_network, response_time, verbose)
                    exists = "no"

            elif error_type == "":
                exists = "error"

            # Save exists flag
            results_site['exists'] = exists

            # Save results from request
            results_site['http_status'] = http_status
            results_site['error_type'] = error_type
            results_site['response_object'] = r
            results_site['response_text'] = response_text
            results_site['response_time_ms'] = response_time

            # Add this site's results into final dictionary with all of the other results.
            results_total[social_network] = results_site

        if not silent_on_error:
            dump_errors()
        return results_total


import sys

def sol():
    N, *arr = map(int, sys.stdin.read().split())
    stack = []
    while N:
        N -= 1
        a = arr[N]
        while stack and stack[-1] <= a:
            stack.pop()
        arr[N] = str(stack[-1]) if stack else "-1"
        stack.append(a)
    print(" ".join(arr))

if __name__ == "__main__":
    sol()


import json
import scrapy
from locations.items import GeojsonPointItem
from locations.hours import OpeningHours

DAY_MAPPING = {
    'MONDAY': 'Mo',
    'TUESDAY': 'Tu',
    'WEDNESDAY': 'We',
    'THURSDAY': 'Th',
    'FRIDAY': 'Fr',
    'SATURDAY': 'Sa',
    'SUNDAY': 'Su',
}

COOKIES = {
            "bm_sz": "04B124C1C96D68082A9F61BAAAF0B6D5~YAAQdjsvF22E8Xl6AQAACr1VfAxPEt+enarZyrOZrBaNvyuX71lK5QPuDR/FgDEWBZVMRhjiIf000W7Z1PiAjxobrz2Y5LcYMH3CvUNvpdS3MjVLUMGwMEBCf9L5nD5Gs9ho2YL8T7Tz7lYvpolvaOlJnKrHyhCFxxk/uyBZ2G/0QrGKLwSaCQShDsz7ink=",
            "_abck": "440E40C406E69413DCCC08ABAA3E9022~-1~YAAQdjsvF26E8Xl6AQAACr1VfAYznoJdJhX7TNIZW1Rfh6qRhzquXg+L1TWoaL7nZUjXlNls2iPIKFQrCdrWqY/CNXW+mHyXibInMflIXJi5VVB/Swq53kABYJDuXYSlCunYvJAzMSr1q12NOYswz134Y8HRNzVWhkb2jMS5whmHxS/v0vniIvS1TQtKjEQlMGzQYmN41CmLX0JobipQhDtUB4VyNwztb2DCAZiqDX8BLwWg7h/DtPd4158qU69hNhayFTgWmD76/MiR8/T536tMmcoRyWLl4fEtP/XUmKOcksuZO7dbfNxXBffTxIXPYwf1eO77LNuZTCQq5kfsGZLJX8ODju2KSjnIF1vdnyHAe98FDIm+hw==~-1~-1~-1"
        }

HEADERS = {
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'accept-encoding': 'gzip, deflate, br',
    'cache-control': 'max-age=0',
    'referer': 'https://www.tesco.com/store-locator/?q=london&qp=london&l=en',
    'sec-ch-ua': '" Not;A Brand";v="99", "Google Chrome";v="91", "Chromium";v="91"',
    'sec-ch-ua-mobile': '?0',
    'sec-fetch-dest': 'document',
    'sec-fetch-mode': 'navigate',
    'sec-fetch-site': 'same-origin',
    'sec-fetch-user': '?1',
    'upgrade-insecure-requests': '1',
    'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.106 Safari/537.36',
    }

class TescoSpider(scrapy.Spider):
    name = "tesco"
    item_attributes = {'brand': "Tesco"}
    allowed_domains = ["www.tesco.com"]
    download_delay = 0.3
    custom_settings = {
        'USER_AGENT': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36',
    }

    def store_hours(self, store_hours):
        opening_hours = OpeningHours()
        store_hours = json.loads(store_hours)

        for day in store_hours:
            closed = day['isClosed']
            if not closed:
                opening_hours.add_range(day=DAY_MAPPING[day['day']],
                                        open_time=str(day['intervals'][0]['start']).zfill(4),
                                        close_time=str(day['intervals'][0]['end']).zfill(4),
                                        time_format='%H%M')

        return opening_hours.as_opening_hours()

    def start_requests(self):
        url = 'https://www.tesco.com/store-locator/sitemap.xml'

        yield scrapy.http.FormRequest(
            url=url,
            method='GET',
            dont_filter=True,
            cookies=COOKIES,
            headers=HEADERS,
            callback=self.parse
            )

    def parse(self, response):
        response.selector.remove_namespaces()
        urls = response.xpath('//url/loc/text()').extract()

        # Exclude location subpages for at-store services
        service_paths = ['petrol-filling-station', 'cafe', 'travel-money', 'f-f-clothing', 'pharmacy']

        for url in urls:
            if not any(service_path in url for service_path in service_paths):

                yield scrapy.http.FormRequest(
                    url=url,
                    method='GET',
                    dont_filter=True,
                    cookies=COOKIES,
                    headers=HEADERS,
                    callback=self.parse_store
                )

    def parse_store(self, response):
        city_page = response.xpath('//span[@class="Hero-title Hero-title--top"]/text()').extract_first()

        if not city_page:  # and is a store page
            store_details = response.xpath('//script[@type="application/json" and contains(text(),"storeID")]/text()').extract_first()
            if store_details:
                store_data = json.loads(store_details)
                ref = store_data['storeID']
                name = store_data['pageName']
                addr_1 = response.xpath('//div[@class="Core-infoWrapper"]//span[@class="Address-field Address-line1"]/text()').extract_first()
                addr_2 = response.xpath('//div[@class="Core-infoWrapper"]//span[@class="Address-field Address-line2"]/text()').extract_first()
                if addr_2:
                    addr_full = ', '.join([addr_1.strip(), addr_2.strip()])
                else:
                    addr_full = addr_1

                properties = {
                    'ref': ref,
                    'name': name,
                    'addr_full': addr_full,
                    'city': response.xpath('//div[@class="Core-infoWrapper"]//span[@class="Address-field Address-city"]/text()').extract_first(),
                    'postcode': response.xpath('//div[@class="Core-infoWrapper"]//span[@class="Address-field Address-postalCode"]/text()').extract_first(),
                    'country': 'GB',
                    'lat': response.xpath('//div[@class="Core-infoWrapper"]//span[@class="Address-coordinates"]/meta[@itemprop="latitude"]/@content').extract_first(),
                    'lon': response.xpath('//div[@class="Core-infoWrapper"]//span[@class="Address-coordinates"]/meta[@itemprop="longitude"]/@content').extract_first(),
                    'phone': response.xpath('//div[@class="Core-infoWrapper"]//span[@itemprop="telephone"]/text()').extract_first(),
                    'website': response.url
                }

                hours = response.xpath('//div[@class="Core-infoWrapper"]//@data-days').extract_first()
                if hours:
                    properties['opening_hours'] = self.store_hours(hours)

                yield GeojsonPointItem(**properties)


import chi

env = 'CartPole-v0'
# env = 'OneRoundDeterministicReward-v0'

@chi.experiment
def dqn_atari(env=env, logdir=""):
  # from PIL import Image
  import numpy as np
  import gym
  import tensorflow as tf
  from gym import wrappers
  from tensorflow.contrib import layers
  from tensorflow.contrib.framework import arg_scope
  from chi.util import in_collections

  chi.set_loglevel('debug')

  env = gym.make(env)
  if hasattr(env.observation_space, 'shape'):
    oshape = env.observation_space.shape
    obsp = env.observation_space
    print(f'Observations in [{obsp.low}, {obsp.high}]')
  else:
    oshape = [1]

  env = wrappers.Monitor(env, logdir+'/monitor')
  # wrappers.SkipWrapper(4)

  if hasattr(env, 'spec') and hasattr(env.spec, 'reward_threshold'):
    print(f'reward threshold: {env.spec.reward_threshold}')

  @chi.model(tracker=tf.train.ExponentialMovingAverage(1-.01))
  def deep_q_network(x: [oshape]):
    with arg_scope([layers.fully_connected],
                   outputs_collections=[tf.GraphKeys.ACTIVATIONS],
                   weights_regularizer=layers.l1_regularizer(.00001)):
      # x = layers.conv2d(x, 32, 8, 4)
      # x = layers.conv2d(x, 64, 4, 2)
      # x = layers.conv2d(x, 64, 3, 1)
      x = layers.fully_connected(x, 32)
      x = layers.fully_connected(x, 32)
      x = layers.fully_connected(x, env.action_space.n, activation_fn=None)
      x = tf.identity(x, name='Q')
      print(f'dqn activations: {chi.activations()}')
      return x

  deep_q_network.optimizer = tf.train.AdamOptimizer(.1)  # .00025
  m = chi.rl.ReplayMemory(500000, oshape, 1)

  @chi.function
  def act(x: [oshape]):
    qs = deep_q_network(x)
    a = tf.argmax(qs, axis=1)
    qm = tf.reduce_max(qs, axis=1)
    layers.summarize_tensor(a)
    return a, qm

  al = "\n".join([f"{v.name} - {in_collections(v)}" for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)])
  print('global vars \n' + al + '\n')

  @chi.function
  def train(o, a: (tf.int32, (None, 1)), r: (None,), t: (tf.bool, (None,)), o2):
    asq = tf.squeeze(a, axis=1)

    q = deep_q_network(o)
    # ac = tf.argmax(q, axis=1)

    with tf.name_scope('compute_targets'):
      q2 = deep_q_network.tracked(o2)
      q_target = tf.where(t, r, r + 0.99 * tf.reduce_max(q2, axis=1))
      q_target = tf.stop_gradient(q_target)

    oh = tf.one_hot(asq, env.action_space.n, 1.0, 0.0, axis=1)
    qs = tf.reduce_sum(q * oh, axis=1, name='q_max')
    td = tf.subtract(q_target, qs, name='td')
    # td = tf.clip_by_value(td, -10, 10)
    mse = tf.reduce_mean(tf.abs(td), axis=0, name='mae')
    # mse = tf.where(tf.abs(td) < 1.0, 0.5 * tf.square(td), tf.abs(td) - 0.5, name='mse_huber')
    # mse = tf.reduce_mean(tf.square(td), axis=0, name='mse')

    with tf.name_scope('minimize'):
      loss = deep_q_network.minimize(mse)

    with tf.name_scope('summarize'):
      print(f'train activations: {chi.activations()}')
      layers.summarize_tensors([td, mse, r, o, a,
                                tf.subtract(o2, o, name='state_dif'),
                                tf.reduce_mean(tf.cast(t, tf.float32), name='frac_terminal'),
                                tf.subtract(tf.reduce_max(q, 1, True), q, name='av_advantage')])
      layers.summarize_tensors(chi.activations())
      layers.summarize_tensors(chi.gradients())
    return loss

  @chi.function
  def log_weigths():
    v = deep_q_network.trainable_variables()
    print(f'log weights {v}')

    f = deep_q_network.tracker_variables
    print(f'log weights EMA {f}')

    difs = []
    for g in v:
      a = deep_q_network.tracker.average(g)
      difs.append(tf.subtract(g, a, name=f'ema/dif{g.name[:-2]}'))

    layers.summarize_tensors(v+f+difs)

  @chi.function
  def log_returns(R, qs):

    layers.summarize_tensors([R, qs, tf.subtract(R, qs, name='R-Q')])

  print('\n VARIABLES: \n')
  al = "\n".join([f"{v.name} - {in_collections(v)}" for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)])
  print(al)

  print('\n START TRAINING: \n')

  training_time = 1000000
  t = 0
  Rs = [0]
  for ep in range(1000000):
    ob = env.reset()
    done = False
    R = 0
    qs = []
    while not done:
      anneal = 1 - t / training_time
      if np.random.rand() < .05 * anneal:
        a = np.random.randint(0, env.action_space.n)
      else:
        # ob = [ob]
        a, qm = act(ob)
        qs.append(qm)

      ob2, r, done, _ = env.step(a)
      r = r
      m.enqueue(ob, a, r, done)

      ob = ob2
      R += r
      if t > 1000:
        train(*m.minibatch(32)[:-1])

      # if (t+1) % 100 == 0:
      #   print(m)
      #   exit()

      if t % 5000 == 0:
        print(f'Return average after {t} timesteps: {np.average(Rs)}')
        Rs = []

      if t % 20000 == 0:
        log_weigths()

      t += 1

    Rs.append(R)
    log_returns(R, qs)

    if t > training_time:
      break

      # dqn = DQNAgent(model=model, nb_actions=nb_actions, policy=policy, window_length=WINDOW_LENGTH, memory=memory,
#                processor=processor, nb_steps_warmup=50000, gamma=.99, delta_range=(-1., 1.),
#                target_model_update=10000, train_interval=4)

#
# INPUT_SHAPE = (84, 84)
# WINDOW_LENGTH = 4
#
#
# class AtariProcessor(Processor):
#     def process_observation(self, observation):
#         assert observation.ndim == 3  # (height, width, channel)
#         img = Image.fromarray(observation)
#         img = img.resize(INPUT_SHAPE).convert('L')  # resize and convert to grayscale
#         processed_observation = np.array(img)
#         assert processed_observation.shape == INPUT_SHAPE
#         return processed_observation.astype('uint8')  # saves storage in experience memory
#
#     def process_state_batch(self, batch):
#         # We could perform this processing step in `process_observation`. In this case, however,
#         # we would need to store a `float32` array instead, which is 4x more memory intensive than
#         # an `uint8` array. This matters if we store 1M observations.
#         processed_batch = batch.astype('float32') / 255.
#         return processed_batch
#
#     def process_reward(self, reward):
#         return np.clip(reward, -1., 1.)
#
# parser = argparse.ArgumentParser()
# parser.add_argument('--mode', choices=['train', 'test'], default='train')
# parser.add_argument('--env-name', type=str, default='Breakout-v0')
# parser.add_argument('--weights', type=str, default=None)
# args = parser.parse_args()
#
# # Get the environment and extract the number of actions.
# np.random.seed(123)
# env.seed(123)
# nb_actions = env.action_space.n

# We patch the environment to be closer to what Mnih et al. actually do: The environment
# repeats the action 4 times and a game is considered to be over during training as soon as a live
# is lost.
# def _step(a):
#     reward = 0.0
#     action = env._action_set[a]
#     lives_before = env.ale.lives()
#     for _ in range(4):
#         reward += env.ale.act(action)
#     ob = env._get_obs()
#     done = env.ale.game_over() or (args.mode == 'train' and lives_before != env.ale.lives())
#     return ob, reward, done, {}
# env._step = _step

# Select a policy. We use eps-greedy action selection, which means that a random action is selected
# with probability eps. We anneal eps from 1.0 to 0.1 over the course of 1M steps. This is done so that
# the agent initially explores the environment (high eps) and then gradually sticks to what it knows
# (low eps). We also set a dedicated eps value that is used during testing. Note that we set it to 0.05
# so that the agent still performs some random actions. This ensures that the agent cannot get stuck.
# policy = LinearAnnealedPolicy(EpsGreedyQPolicy(), attr='eps', value_max=1., value_min=.1, value_test=.05,
#                               nb_steps=1000000)

# dqn = DQNAgent(model=model, nb_actions=nb_actions, policy=policy, window_length=WINDOW_LENGTH, memory=memory,
#                processor=processor, nb_steps_warmup=50000, gamma=.99, delta_range=(-1., 1.),
#                target_model_update=10000, train_interval=4)

# The trade-off between exploration and exploitation is difficult and an on-going research topic.
# If you want, you can experiment with the parameters or use a different policy. Another popular one
# is Boltzmann-style exploration:
# policy = BoltzmannQPolicy(tau=1.)
# Feel free to give it a try!


from tkinter import *
from time import *
from random import *

def ins_with_bars(initial):
    array = [0] + initial
    for i in range(1,len(array)):
        if array[i-1] > array[i]:
            array[0] = array[i]
            j = i - 1
            while array[j] > array[0]:
                array[j+1] = array[j]
                j -= 1
            array[j+1] = array[0]
            #print(array)
    initial = array[1:]
    return initial

tm = perf_counter() # Время, когда началось высчитывание

#random.randrange(start, stop, step) - возвращает случайно выбранное число из последовательности.
#random.randint(A, B) - случайное целое число N, A ≤ N ≤ B.


#!/bin/env python
"""Build DAOS Control Plane"""
#pylint: disable=too-many-locals
import os
import sys
from os.path import join, isdir
from os import urandom
from binascii import b2a_hex

Import('env', 'prereqs', 'DAOS_VERSION', 'CONF_DIR', 'gurt_lib')

GO_COMPILER = 'go'
MIN_GO_VERSION = '1.12.0'
GO_BIN = env.get("GO_BIN", env.WhereIs(GO_COMPILER))

def check_dir_exists(path):
    """
    Check if path points to an existing directory. If it is a file but not a
    directory, delete it.
    """
    if not isdir(path):
        # Not a directory - clear out any file
        if os.path.exists(path):
            os.unlink(path)
        return False
    return True

def get_install_src_dir(repopath, name):
    """Get the Gopath-based directory to run 'go install' on"""
    return join(repopath, "src", "control", "cmd", name)

def get_install_bin_path(benv, name):
    """Get the final installation location for a binary with a given name"""
    return join(benv.subst("$PREFIX"), "bin", name)

#pylint: disable=too-many-arguments
def install_go_bin(denv, gosrc, libs, name, install_name):
    """
    Build a Go binary whose source is under directory 'name' and install it
    as 'install_name'.
    libs should be a list of scons-built libraries, or None if none are needed.
    """
    # Repository path shared by DAOS Go package import paths
    repopath = join("github.com", "daos-stack", "daos")

    mod_src = join(gosrc, "cmd", name, "main.go") # Module src
    install_src = get_install_src_dir(repopath, name)
    install_bin = get_install_bin_path(denv, install_name)
    build_bin = join('$BUILD_DIR', 'src/control/bin', name)
    src = [mod_src]
    if libs is not None:
        src.extend(libs)
    def gen_build_id():
        """generate a unique build id per binary for use by RPM
           https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID"""
        buildid = b2a_hex(urandom(20))
        if isinstance(buildid, str):
            # Python 2 is a str already, using decode() here transforms it
            # into a 'unicode' type
            return '0x' + buildid
        # Python 3 is a bytes, decode() it to a str
        return '0x' + buildid.decode()

    def go_ldflags():
        return ' '.join(['-ldflags',
                         '"-X github.com/daos-stack/daos/src/control/build.DaosVersion=%s' % DAOS_VERSION,
                         '-X github.com/daos-stack/daos/src/control/build.ConfigDir=%s' % CONF_DIR,
                         '-B %s"' % gen_build_id()])
    # Must be run from the top of the source dir in order to
    # pick up the vendored modules.
    denv.Command(build_bin, src + gurt_lib,
                 'cd %s; %s build -mod vendor -v %s -o %s %s' %
                 (gosrc, GO_BIN, go_ldflags(), build_bin, install_src))
    # Use the intermediate build location in order to play nicely
    # with --install-sandbox.
    denv.InstallAs(install_bin, build_bin)
    return install_bin
#pylint: enable=too-many-arguments

def configure_go(denv):
    """Do Go compiler checks"""
    if GetOption('help') or GetOption('clean'):
        return
    def check_go_version(context):
        """Check GO Version"""
        context.Display('Checking for Go compiler in $PATH... ')
        if GO_BIN:
            context.Display(GO_BIN + '\n')
        else:
            context.Result(0)
            return 0

        context.Display('Checking %s version... ' % GO_BIN)
        out = os.popen('%s version' % GO_BIN).read()
        if len(out.split(' ')) < 3:
            context.Result('failed to get version from "%s"' % out)
            return 0

        # go version go1.2.3 linux/amd64
        go_version = out.split(' ')[2].replace('go', '')
        if len([x for x, y in
                zip(go_version.split('.'), MIN_GO_VERSION.split('.'))
                if int(x) < int(y)]) > 0:
            context.Result('%s is too old (min supported: %s) '
                           % (go_version, MIN_GO_VERSION))
            return 0
        context.Result('%s' % go_version)
        return 1

    conf = Configure(denv, custom_tests={'CheckGoVersion': check_go_version})
    # pylint: disable=no-member
    if not conf.CheckGoVersion():
        print('no usable Go compiler found (yum install golang?)')
        sys.exit(1)
    conf.Finish()
    # pylint: enable=no-member

def scons():
    """Execute build"""
    env.AppendUnique(LIBPATH=[Dir('.')])

    denv = env.Clone()

    if denv.get("COMPILER") == 'covc':
        denv.Replace(CC='gcc', CXX='g++')

    # if SPDK_PREFIX differs from PREFIX, copy dir so files can be accessed at
    # runtime
    prefix = denv.subst("$PREFIX")
    sprefix = denv.subst("$SPDK_PREFIX")
    if sprefix not in ["", prefix]:
        def install_dir(ienv, srcdir, _destdir):
            """walk a directory and install targets"""
            for root, _dirs, files in os.walk(srcdir):
                dest_root = os.path.relpath(root, sprefix)
                for fname in files:
                    full_name = os.path.join(root, fname)
                    dest = os.path.join(prefix, dest_root)
                    ienv.Install(dest, full_name)

        for _dir in [join("share", "spdk", "scripts"), join("include", "spdk")]:
            target = join(prefix, _dir)
            source = join(sprefix, _dir)
            install_dir(denv, source, target)

    configure_go(denv)

    # Version-controlled DAOS Go source directory src/control
    gosrc = Dir('.').srcnode().abspath
    cartpath = Dir('#/src/cart/src/include').srcnode().abspath

    prereqs.require(denv, 'ofi', 'hwloc')

    denv.AppendENVPath("CGO_LDFLAGS",
                       denv.subst("$_LIBDIRFLAGS "
                                  "$_RPATH"),
                       sep=" ")

    denv.AppendENVPath("CGO_CFLAGS", denv.subst("$_CPPINCFLAGS"),
                       sep=" ")

    agentbin = install_go_bin(denv, gosrc, None, "daos_agent", "daos_agent")
    dmgbin = install_go_bin(denv, gosrc, None, "dmg", "dmg")
    drpcbin = install_go_bin(denv, gosrc, None, "drpc_test", "hello_drpc")

    AlwaysBuild([agentbin, dmgbin, drpcbin])

    SConscript('lib/spdk/SConscript', exports='denv')

    senv = denv.Clone()
    prereqs.require(senv, 'pmdk', 'spdk', 'ofi', 'hwloc')

    cgolibdirs = senv.subst("-L$BUILD_DIR/src/control/lib/spdk "
                            "-L$BUILD_DIR/src/cart/src/gurt "
                            "-L$BUILD_DIR/src/cart/src/cart "
                            "-L$SPDK_PREFIX/lib "
                            "-L$OFI_PREFIX/lib -L$ISAL_PREFIX/lib $_RPATH")
    # Explicitly link RTE & SPDK libs for CGO access (need for clang build).
    senv.AppendENVPath("CGO_LDFLAGS", cgolibdirs + \
                       " -lspdk_env_dpdk -lspdk_thread -lspdk_bdev" + \
                       " -lspdk_copy -lrte_mempool  -lrte_mempool_ring" + \
                       " -lrte_bus_pci -lrte_pci -lrte_ring -lrte_mbuf" + \
                       " -lrte_eal -lrte_kvargs -lspdk_bdev_aio" + \
                       " -lspdk_bdev_nvme -lspdk_bdev_malloc -lspdk_conf" + \
                       " -lspdk_blob -lspdk_nvme -lspdk_util -lspdk_json" + \
                       " -lspdk_jsonrpc -lspdk_rpc -lspdk_trace" + \
                       " -lspdk_sock -lspdk_log -lspdk_notify" + \
                       " -lspdk_blob_bdev -lnuma -ldl -lisal", sep=" ")

    senv.AppendENVPath("CGO_CFLAGS",
                       senv.subst("-I$SPDK_PREFIX/include "
                                  "-I$OFI_PREFIX/include"),
                       sep=" ")

    # Copy setup_spdk.sh script to be executed at daos_server runtime.
    senv.Install(join('$PREFIX', 'share/daos/control'),
                 join(gosrc, 'server/init/setup_spdk.sh'))

    serverbin = install_go_bin(senv, gosrc, [denv.nvmecontrol], "daos_server",
                               "daos_server")
    adminbin = install_go_bin(senv, gosrc, [denv.nvmecontrol], "daos_admin",
                              "daos_admin")

    AlwaysBuild([serverbin, adminbin])

if __name__ == "SCons.Script":
    scons()


# encoding: utf-8
# module wildfowl_vision

def rgb2bgr(rgb):
    return list(reversed(rgb))

def contour_len(cnr):
    l = 0
    for i in range(0, len(cnr) - 1):
        x1, y1 = cnr[i][0]
        x2, y2 = cnr[i + 1][0]
        l += pow(pow(x1 - x2, 2) + pow(y1 - y2, 2), 0.5)
    return l

def filter_long_contours(contours):
    res = list()
    for i in range(0, len(contours)):
        if contour_len(contours[i]) >= 100:
            res.append(contours[i])
    return res


import pandas as pd
import numpy as np
from nltk.tokenize import TreebankWordTokenizer
import string
from konlpy.tag import Komoran,Kkma,Okt,Hannanum,Mecab,Twitter
from wordcloud import STOPWORDS
myframe = pd.read_csv('../concat_data/updatae_csv.csv')
komoran = Komoran()
tokenizer = TreebankWordTokenizer()
stopword = set(STOPWORDS)
review = myframe['text'][0]

tokens = komoran.pos(review)

token_list = set([ x[0] for x in tokens if x[1] in ('NNG','NNP')])

# print(token_list)
# print(tokenizer.tokenize(review))
import re
regex1 = '^[\d가-힣a-zA-Z]*$'
regex2 = '^[\d]*$'
pattern1 = re.compile(regex1)
pattern2 = re.compile(regex2)
token_set = set()
# for idx in range(0,61):
#     review = myframe['text'][idx]
#     review = review.replace(u'\\xa0','')
#
#     # review = review.replace(u'\\n','').encode('utf-8')
#     tknize = tokenizer.tokenize(review)
#
#
#     tknize_st = [x for x in tknize if pattern1.match(x) and not x.isnumeric()]
#     print(idx, tknize_st)
#     token_set.update(tknize_st)
# print(token_set)
# print(len(token_set))

kkma = Kkma()
okt = Okt()
hananum = Hannanum()
twit = Twitter()
# mecab = Mecab()

hangul_stopword = [x.strip() for x in open('../concat_data/hangle_stop_word.txt', 'r', encoding='utf-8').readlines()]

token_set = set()
print(string.punctuation)  # !"#$%&'()*+,-./:;<=>?@[\]^_`{|}~
pt = re.compile("!\"#")
pattern = re.compile('[\U00010000-\U0010ffff]',flags=re.UNICODE)
for idx in range(0,61):
    review = myframe['text'][idx]
    review = pattern.sub(r' ',review)
    tokens = twit.pos(review,norm=True)
    # 불용어 처리
    tokens = [ x[0].replace(u'\u3000',u' ').replace(u'\xa0',u' ').translate(str.maketrans('', '', string.punctuation)) for x in tokens if x not in hangul_stopword]
    tokens = [ x for x in tokens if len(x)>1]
    token_set.update(tokens)
    print(idx,tokens)

print(len(token_set))
pat = re.compile('^[\d\r가-힣]*$')
bul = [x for x in token_set if not pat.match(x) or len(x)==1]
print(bul)

print(string.punctuation)
print(string.ascii_letters)
# ['※', '&', '!', "'", ')', '–', '-', '=', 'i', '(', '.', 'ㄴ', '☞', '~', '·', 'ㅁ', 'ㄹ', ':', '/', 'P', ',', '%', '"', '→']
# ['4월\xa0밸런스', '매', '얻고\u3000\xa0', '스테이지\u3000·', '그랩,', '힘', '넘', '3초\xa0→', "30초'\xa0게", "증가\xa0'3\xa0→",
#  '·\xa0업데이트', '돕', "%'-", '모드와\xa0새로운', '‘1.6초', '모델\xa0(특수', '회전기\u3000\u3000회전하', "50'\u3000커브볼(스타",
#  '승리\xa0', '램프\u3000\u3000주위', '폭풍\u3000\u300012초', '‘300', '스니커즈\u3000\u3000발', '증가\xa0‘4,600', '150\xa0트로피',
#  '쉘리\u3000쉘', '질주\u3000\u3000리코', '·\xa0미스터', '폭풍이\xa0아군', '–', '피루엣\u3000\u3000특수', '끊', '5초’-', '새', '·\xa0희귀',
#  '수정\u3000·', '엠즈\u3000후원', '상점\u3000·', '기준으로\xa0매치메이킹', '·\xa0재키\xa0-', '변경\u3000특수', '프랭크\u3000노이즈', '별',
#  '700’-', '틱\u3000미니', '휘', '수정\u3000·\xa0새로운', '페니\u3000약탈', "'1840\xa0→\xa0", "'640\xa0→", '자세(스타', '포크아웃\u3000·',
#  "'70\xa0→", '그랩(스타', '증폭\u3000\u3000특수', '요', '지역\u3000·', '길', '선', '0초’\u3000오토매틱', '29일(화)부터', '0.3초',
#  '보석\xa0\u3000\xa0에이전트', '증가\xa0‘40', '타', "'3500\xa0→", '테마\u3000·', '보석\xa0\u3000\xa0늑대', '로사의\xa0몸',
#  "'260\xa0→", "'420\xa0→", '25%’-', '‘100%', '0.5개', '은신술\u3000\u3000은신', "'240\xa0→", '리모델링\u3000·\xa0페니',
#  '내역\u3000·\xa0모티스', '600’\u3000그림자', '-\xa0모티스\u3000전속력', '얻고\xa0', '‘8%', '‘격려', '잃', '시\xa0재장전',
#  '곁', '론', '하이스트\u3000\u3000삭제', '·\xa08비트\xa0-', '5월\xa0밸런스', '프랭크\u3000파워', '수정\u3000·\xa0바운티', '→',
#  '지뢰\u3000\u3000지면', '·\xa0니타\xa0-', '파이퍼\u3000팡팡팡(특수', '리코\u3000통통탄(일반', '브롤러\u3000\u3000로사',
#  '3,200’\xa0오류', '(빅', '피루엣(스타', '0.5초', '밸런스\xa0-', '감소\xa0‘5,500\xa0→', '에', '1,000’-\xa0대릴\u3000강철',
#


#%%

import numpy as np
import numpy.matlib as ml

from scipy import signal
import scipy.optimize
from scipy.stats import invgauss
import scipy.stats as sct

from .utils import idm
from .utils.misc import merge_dict

import matplotlib.pyplot as plt

from plotly import tools
import plotly.graph_objs as go
from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot
init_notebook_mode(connected=True)

class vel_spectra:
    """
    Velocity spectra class.
    Derived from the method and code of Bluteau, Jones and Ivey (2011), itself following Ruddick et al (2000).

    Ultimately the Batchelor spectrum method should be coded (Ruddick et al 2000), at which point a
    generic spectra superclass should be created which contains common features.
    """

    window = 'hamming'
    verboso = True
    verbosissimo = False

    visc = 1e-6  # viscosity (m2s-1)

    def __init__(self, u, U, fs, comp = 1, n_blocks=10, ds=1, nbands=1, fbands=32, **kwargs):
        """
        Init function. Will break the entire spectrum into

        Required arguments:

            u - the timeseries
            fs - the sample frequency (Hz)
            U - mean streamwise velocity (m/s)

        Optional arguments:

            comp - velocity component ( [1-streamwise] | [2-crossstream] | [3-vertical] )
            n_blocks
            ds
            nbands
            fbands

        """

        self.__dict__.update(**kwargs)

        self.u = u
        self.U = U
        self.fs = fs
        self.comp = comp

        self.n_blocks = n_blocks
        self.ds = ds
        self.nbands = nbands
        self.fbands = fbands

        self.f = np.nan
        self.Pxx = np.nan
        self.dof = 30                   # Should not be hard coded

        ## SEARCH ATTRIBUTES
        self.decades    = 0.7           # Final number of decades in search
        self.NS = []                    # This should be an attribute like U, don't use for now
        self.fastR = 1.05
        self.t_fast = False             # Option to run fast version
        self.min_r2 = 0.6               # Minimum r2 for accepting an ISR fit
        self.slope_range=(1/3)*np.array((-1, 1))-5/3; # Acceptable slope range for accepting an ISR fit
        ## SEARCH ATTRIBUTES

        self.plotlib = 'plotly'

        self.testing = True             # Toggle some features for running tests against original code

        if np.any(np.isnan(u)):
            if self.verboso:
                print("Initialising with no u. Must be done later, or f and Pxx must be specified later.")
            pass
        elif len(u)/fs < n_blocks:
            raise(BadBlockLength)

    def check_window(self):

        if self.window == 'hanning':
            pass
        elif self.window == 'hamming':
            pass
        else:
            raise(Exception)

        if self.verboso:
            print('Window OK!!')

    def spec_welch(self):
        """
        Calculate spectrum using Welch's method.

        Saves f and Pxx in Hz and Energy per Hz

        """

        self.check_window()
        u = signal.detrend(self.u)

        nperseg=np.floor(2*len(u)/(self.n_blocks+1)) # Force 50% overlap
        nperseg=2**np.ceil(np.log2(np.abs(nperseg))) # Force next power of two for now as original code does so

        [f, Pxx] = signal.welch(u, self.fs, window=self.window, nperseg=nperseg)

        self.f = f
        self.Pxx = Pxx

        if self.verboso:
            print('Spectrum calculated')
            print('nperseg of {0}'.format(nperseg))
            print('{0} points in the spectrum'.format(len(f)))

        if self.fbands > 0 and self.nbands > 1:

            self.spec_band_average()

        return

    def spec_band_average(self):

        print('Band averaging now.')
        raise IDMException('Yet to code band averaging. ')

    def spec_search(self):
        """
        Search existing spectra in rad/s to find inertial subrange, and then dissipation using MLE.

        NOTE: this library is not performing a second loop in search of the best portion of spectrum, as done in original code. This was not described in BIJ11.

        Inputs:
            dof - degrees of freedom. This should be calculable above and storeable, to review.

        Things which could be made into kwargs:
            lowfreqlim
            highfreqlim
        """
        # [epsilon_3,  misfit, best_ind, allmisfit, ax] = idm_SpectraSearch(U, Sxx_64_rps , n_64_rps, 3, 30, 'plot', false, 'fast', true);

        print('Have hard coded dof. Change this!')
        print("Should code the speed up option.")

        fast_opt = False

        # End of inputs section
        accel = 1
        if fast_opt:
            accel = 10

        dec_lin = 10**self.decades

        f_rps, Pxx_rps = self.spec_toRadiansPerSec()

        n = len(f_rps)

        mad_best = np.inf # Initial best fit error
        epsilon_best = np.nan

        ind = [i for i in np.arange(0, n) if f_rps[i] > 0]
        f_rps_clean = f_rps[ind]
        Pxx_rps_clean = Pxx_rps[ind]

        # If using static lowfreqlim and highfreqlim we don't need to usereindex here
        highfreqlim = np.max(f_rps_clean)     # Noise floor
        lowfreqlim = f_rps_clean[0]

        spec_fit = {}
        spec_fit["f_rps"] =  f_rps_clean
        spec_fit["P_xx_rps"] =  Pxx_rps_clean
        self.spec_fit = spec_fit

        n_clean = len(f_rps_clean)

        # First loop done forwards using the full range of self.decades
        for i in np.arange(0, n_clean, accel):

            if i/50 == np.round(i/50) and i>0 and self.verboso:
                print("Scanning portion {0}. Current epsilon: {1}".format(i, epsilon_best))
            pdei = self.get_pdei(i, dec_lin)

            if pdei is None: # Could do one more
                print('Breaking')
                break

            mle_data = idm.epsilon_calcs(f_rps_clean[i:pdei], Pxx_rps_clean[i:pdei], self.U, self.dof, comp=self.comp, verbosissimo=self.verbosissimo)

            if i > 0:
                all_mle_data = merge_dict(all_mle_data, mle_data)
            else:
                all_mle_data = mle_data

            happy = self.idm_misfit_check(mle_data)
            if not happy:
                continue
            elif mle_data['mad'] < mad_best:
                mad_best = mle_data['mad']
                ind_best = np.arange(i, pdei)
                epsilon_best = mle_data['epsilon']
                n_best = len(all_mle_data['epsilon'])
                mean_f_rps_best = mle_data['mean_f_rps']

        if self.verboso:
            print("NOTE: this library is not performing a second loop in search of the best portion of spectrum, as done in original code. This was not described in the paper.")

        mle_data = {}
        mle_data['n_best'] = n_best
        mle_data['epsilon_best'] = epsilon_best
        mle_data['ind_best'] = ind_best
        mle_data['mean_f_rps_best'] = mean_f_rps_best
        mle_data['all_mle_data'] = all_mle_data

        self.mle_data = mle_data

    def idm_misfit_check(self, misfit):
        """
        Determine whether we are happy with this portion of spectra based on the misfit criteria.

        Criteria used:
            r2
            slope

        Criteria not used:
            mad - I don't know why this is not used. In CB paper but not code.
            B, I - I don't know why this is not used. In CB paper but not code.
        """

        check_r2 = True
        check_slope = True

        if check_r2:
            if misfit['r2'] < self.min_r2:
                if self.verbosissimo:
                    print('Failed on r2')
                return False

        if check_slope:
            if misfit['slope'] < self.slope_range[0] or misfit['slope'] > self.slope_range[1]:
                if self.verbosissimo:
                    print('Failed on slope')
                return False

        return True

    def get_pdei(self, pd_start, dec_lin):
        """
        Get end index of a partial decade for a given start index. Gives the pythonic index to be used in slicing directly.
        """

        spec_fit = self.spec_fit
        n_clean = len(spec_fit["f_rps"])

        pdei = [i for i in np.arange(0, n_clean) if spec_fit["f_rps"][i]>dec_lin*spec_fit["f_rps"][pd_start]] # Partial decade end index

        if len(pdei) == 0: # Don't have a decade left
            return None
        else:
            return pdei[0]+1

    def get_pdsi(self, pd_end, dec_lin):
        """
        Get start index of a partial decade for a given end index. Gives the pythonic index to be used in slicing directly.
        """

        spec_fit = self.spec_fit
        n_clean = len(spec_fit["f_rps"])

        pdsi = [i for i in np.arange(0, n_clean) if spec_fit["f_rps"][i]>dec_lin*spec_fit["f_rps"][pd_start]] # Partial decade end index

        if len(pdsi) == 0: # Don't have a decade left
            return None
        else:
            return pdsi[0]

    def get_max_mad(self):

        return 2*(2/self.dof)**(1/2)

    def get_mle_best(self):

        n_best = self.mle_data['n_best']
        epsilon_best = self.mle_data['epsilon_best']
        ind_best = self.mle_data['ind_best']
        mean_f_rps_best = self.mle_data['mean_f_rps_best']

        return n_best, epsilon_best, ind_best, mean_f_rps_best

    def spec_toRadiansPerSec(self):
        """
        Converts the spectra stored in Hz and Energy per Hz to rad/s
        """

        Pxx_rps = self.Pxx/(2*np.pi)
        f_rps = self.f*2*np.pi

        return f_rps, Pxx_rps

    def spec_toRadiansPerMetre(self):
        """
        Converts the spectra stored in Hz and Energy per Hz to rad/m space
        """

        f, Pxx = self.spec_toRadiansPerSec()

        Pxx_rpm = Pxx*self.U
        f_rpm = f/self.U

        return f_rpm, Pxx_rpm

    def spec_quickplot(self):

        print('Plotting spectrum')

        f_units = 'Hz'
        f_units = 'rps'
        if f_units.lower() == 'hz':
            Pxx = self.Pxx
            f = self.f
            ylab = 'P{0}{0} (m2s-2/Hz)'.format(self.comp)
            xlab = 'f (Hz)'

        elif f_units.lower() == 'rps':

            f, Pxx = self.spec_toRadiansPerSec()

            ylab = 'P{0}{0} (m2s-2/(rad.s-1))'.format(self.comp)
            xlab = 'f (rad.s-1)'

        tit = 'u{0} Power Spectrum'.format(self.comp)

        if self.plotlib.lower() == 'matplotlib':

            fig, axes = plt.subplots(nrows=1, figsize=(8, 8))
            axes = [axes]

            axes[0].plot(f, Pxx)

            axes[0].set_xlabel(xlab)
            axes[0].set_ylabel(ylab)

            axes[0].set_yscale('log')

        elif self.plotlib.lower() == 'plotly':

            fig = tools.make_subplots(rows=1, cols=1)

            trace1 = {'mode': 'lines+markers', "x": f, "y": Pxx, "name": 'Welch Spectrum', 'marker' : dict(color = 'rgb(17, 157, 255)')}

            fig.append_trace(trace1, 1, 1)

            fig['layout'].update(title=tit)

            fig['layout']['yaxis1'].update(title=ylab, type='log', range = [-9, -3])
            fig['layout']['xaxis1'].update(title=xlab, type='log', range = [-2, 4])

            plot(fig)

    def BIJ11_fig4(self, savename=None):

        print('Plotting MLE calcs')

        f_units = 'rps'

        f_rps, Pxx_rps = self.spec_toRadiansPerSec()

        ylab = 'P{0}{0} (m2s-2/(rad.s-1))'.format(self.comp)
        # ylab = '$P_{{0}{0}} (m^2s^{-2}/(rad.s^{-1}))$'.format(self.comp)
        ylab = '$P_{ii} (m^2s^{-2}/(rad.s^{-1}))$'
        xlab = '$f (rad.s^{-1})$'

        tit = '$u_{0} Power Spectrum$'.format(self.comp)

        print('Changed')
        fig, axes = plt.subplots(nrows=5, figsize=(8, 8))
        # axes = [axes]

        axes[0] = plt.subplot2grid((6, 3), (0, 0), colspan=3, rowspan=2)
        axes[1] = plt.subplot2grid((6, 3), (2, 0), colspan=3, rowspan=1)
        axes[2] = plt.subplot2grid((6, 3), (3, 0), colspan=3, rowspan=1)
        axes[3] = plt.subplot2grid((6, 3), (4, 0), colspan=3, rowspan=1)
        axes[4] = plt.subplot2grid((6, 3), (5, 0), colspan=3, rowspan=1)

        xl = [-2, 3]

        n_best, epsilon_best, ind_best, mean_f_rps_best = self.get_mle_best()
        Pxx_rps_model = idm.isr_theoretical(f_rps[ind_best], epsilon_best, self.U, self.comp)

        axes[0].plot(f_rps, Pxx_rps)
        axes[0].plot(f_rps[ind_best], Pxx_rps_model, color='k')
        # axes[0].set_xlabel(xlab)
        axes[0].set_ylabel(ylab)
        axes[0].set_yscale('log')
        axes[0].set_xscale('log')
        axes[0].set_xlim(10**xl[0], 10**xl[1])
        axes[0].set_ylim(10**-9, 10**-3)
        axes[0].grid()
        axes[0].text(10**xl[0], 10**-8.5, '$\epsilon = {0:0.2e} m^2s^{{-2}}$'.format(epsilon_best))

        print(self.mle_data['all_mle_data'].keys())
        axes[1].plot(self.mle_data['all_mle_data']['mean_f_rps'], self.mle_data['all_mle_data']['epsilon'])
        axes[1].plot(self.mle_data['all_mle_data']['mean_f_rps'][n_best], self.mle_data['all_mle_data']['epsilon'][n_best], marker='o')
        axes[1].set_ylabel('$\epsilon (m^2s^{-2})$')
        axes[1].set_yscale('log')
        axes[1].set_xscale('log')
        axes[1].set_xlim(10**xl[0], 10**xl[1])
        axes[1].grid()

        axes[2].plot(self.mle_data['all_mle_data']['mean_f_rps'], self.mle_data['all_mle_data']['slope'])
        axes[2].plot(self.mle_data['all_mle_data']['mean_f_rps'][n_best], self.mle_data['all_mle_data']['slope'][n_best], marker='o')
        axes[2].plot([10**xl[0], 10**xl[1]], [-5/3, -5/3], color='k', linestyle='dashed')
        axes[2].plot([10**xl[0], 10**xl[1]], [self.slope_range[0], self.slope_range[0]], color='k', linestyle=':')
        axes[2].plot([10**xl[0], 10**xl[1]], [self.slope_range[1], self.slope_range[1]], color='k', linestyle=':')
        axes[2].set_ylabel('$slope ()$')
        axes[2].set_xscale('log')
        axes[2].set_ylim(-4, 1)
        axes[2].set_xlim(10**xl[0], 10**xl[1])
        axes[2].grid()

        axes[3].plot(self.mle_data['all_mle_data']['mean_f_rps'], self.mle_data['all_mle_data']['mad'])
        axes[3].plot(self.mle_data['all_mle_data']['mean_f_rps'][n_best], self.mle_data['all_mle_data']['mad'][n_best], marker='o')
        axes[3].plot([10**xl[0], 10**xl[1]], [self.get_max_mad(), self.get_max_mad()], color='k', linestyle='dashed')
        axes[3].set_ylabel('$MAD ()$')
        axes[3].set_yscale('log')
        axes[3].set_xscale('log')
        axes[3].set_ylim(10**-1, 10**0)
        axes[3].set_xlim(10**xl[0], 10**xl[1])
        axes[3].grid()

        axes[4].plot(self.mle_data['all_mle_data']['mean_f_rps'], self.mle_data['all_mle_data']['r2'])
        axes[4].plot(self.mle_data['all_mle_data']['mean_f_rps'][n_best], self.mle_data['all_mle_data']['r2'][n_best], marker='o')
        axes[4].plot([10**xl[0], 10**xl[1]], [self.min_r2, self.min_r2], color='k', linestyle='dashed')
        axes[4].set_ylabel('$r^2 ()$')
        # axes[4].set_yscale('log')
        axes[4].set_xscale('log')
        axes[4].set_ylim(-1, 1)
        axes[4].set_xlim(10**xl[0], 10**xl[1])
        axes[4].grid()

        plt.tight_layout(pad=0.5, w_pad=0.5, h_pad=1.0)

        if not savename is None:
            plt.savefig(savename)

class BadBlockLength(Exception):
    pass

class BadInputs(Exception):
    pass

class IDMException(Exception):

    pass

#%%


"""
CP1404/CP5632 - Practical
Answer the following questions:
1. When will a ValueError occur?
2. When will a ZeroDivisionError occur?
3. Could you change the code to avoid the possibility of a ZeroDivisionError?
"""

try:
    numerator = int(input("Enter the numerator: "))
    denominator = int(input("Enter the denominator: "))
    while denominator == 0:
        print("Cannot divide by zero!")
        denominator = int(input("Enter non-zero denominator: "))
    fraction = numerator / denominator
    print(fraction)
except ValueError:
    print("Numerator and denominator must be valid numbers!")
except ZeroDivisionError:
    print("Cannot divide by zero!")
print("Finished.")

# Value Errors will occur when instead of an integer or float a different character is being input.
# Zero Division error occur when the second input (denominator) is a 0.


"""
Unit tests for crossref.py
"""

# pylint: disable=no-member, missing-docstring, len-as-condition
import logging

import pytest

from bibliom.publication_objects import Paper
from bibliom.crossref import *

@pytest.mark.usefixtures('class_manager')
class TestCrossref():
    def test_cr_works(self):
        logging.getLogger('bibliom.pytest').debug('-->TestCrossref.test_cr_works')
        response = cr_works(ids='10.1016/j.ijhydene.2016.06.178')
        assert(
            response['message']['title'][0] ==
            'RETRACTED: Plasma equilibrium reconstruction for the nuclear ' +
            'fusion of magnetically confined hydrogen isotopes'
        )

    def test_parse_response(self):
        logging.getLogger('bibliom.pytsest').debug('-->TestCrossref.test_parse_response')
        response = cr_works(ids='10.1016/j.ijhydene.2016.06.178')
        paper = parse_response(response)
        assert isinstance(paper, Paper)
        assert(
            paper.title ==
            'RETRACTED: Plasma equilibrium reconstruction for the nuclear ' +
            'fusion of magnetically confined hydrogen isotopes'
        )


with open("test.txt", "r") as t:
    with open("actual.txt", "r") as a:
        t = t.read()
        a = a.read()

        print(len(t), len(a))
        '''
        l = min(len(t), len(a))
        for i in range(l):
            if (t[i] != a[i]):
                print(i)
        '''


import datetime
from dateutil.relativedelta import relativedelta

from disease import Disease
from practioner_analysis.cdss_practitioners import cdss_practitioners
from practioner_analysis.medication_rate import MedicationRate
from practioner_analysis.plot import plot_data, group_data, plot_difference, plot_medication_breakdown

def get_month_bins(start, end, bin_size):
    date_bins = []
    cur_bin = start
    while cur_bin < end:
        date_bins.append(cur_bin)
        cur_bin += relativedelta(months=bin_size)

    date_bins.append(cur_bin)

    return date_bins

def get_practitioners(patients, spec=None):
    practitioners = set()

    for patient in patients.values():
        for diagnosis in patient.diagnoses.iter_diagnoses():
            if not diagnosis.practitioner:
                continue
            if diagnosis.disease.spec == spec or spec is None:
                practitioners.add(diagnosis.practitioner)

    return practitioners

def get_diagnosis_tuples(patients, disease=None):
    tuples = []
    for patient in patients.values():
        if disease:
            if disease not in patient.diagnoses:
                continue
            for diagnosis in patient.diagnoses[disease]:
                tuples.append((patient, diagnosis))
        else:
            for diagnosis in patient.diagnoses.iter_diagnoses():
                tuples.append((patient, diagnosis))

    return tuples

def analyze_practitioners(patients, start, end, meds_start_with, bin_months=1, **kwargs):
    print("analyzing practitioners...")

    spec, diag = kwargs.get("spec"), kwargs.get("diag")
    practitioners = get_practitioners(patients, spec=spec)

    if spec is None or diag is None:
        disease = None
    else:
        disease = Disease(spec, diag)

    patient_diagnosis_tuples = get_diagnosis_tuples(patients, disease=disease)
    patient_diagnosis_tuples.sort(key=lambda tup: tup[1])

    date_bins = get_month_bins(start, end, bin_months)
    data = {k: {d: MedicationRate() for d in date_bins} for k in practitioners}
    i = 0

    correct_before = []
    correct_after_CDSS = []
    correct_after_no_CDSS = []

    for patient, diagnosis in patient_diagnosis_tuples:
        if diagnosis.practitioner not in practitioners:
            continue
        if diagnosis.start_date < start:
            continue
        if diagnosis.start_date > end:
            break

        while diagnosis.start_date > date_bins[i + 1]:
            i += 1

        score = patient.calculate_chads_vasc(diagnosis.start_date)
        medication = patient.has_medication_group(meds_start_with, diagnosis.start_date, which=True) \
                     or patient.has_medication_group(meds_start_with, diagnosis.start_date + relativedelta(days=+1),
                                                     which=True)

        data[diagnosis.practitioner][date_bins[i]].update(score, medication)

        # Before April 2014 the medication data is incomplete so exclude it
        if diagnosis.start_date < datetime.date(2014, 4, 1):
            continue

        correct = 1 if (score >= 2 and medication) or (score < 2 and not medication) else 0
        if diagnosis.start_date < datetime.date(2015, 7, 1):
            correct_before.append(correct)
        else:
            if diagnosis.practitioner in cdss_practitioners:
                correct_after_CDSS.append(correct)
            else:
                correct_after_no_CDSS.append(correct)

    rate = 0 if len(correct_before) == 0 else sum(correct_before) / len(correct_before)
    print("Correct before:  {} / {} = {}".format(sum(correct_before), len(correct_before), rate))

    rate = 0 if len(correct_after_no_CDSS) == 0 else sum(correct_after_no_CDSS) / len(correct_after_no_CDSS)
    print("Correct no CDSS: {} / {} = {}".format(sum(correct_after_no_CDSS), len(correct_after_no_CDSS), rate))

    rate = 0 if len(correct_after_CDSS) == 0 else sum(correct_after_CDSS) / len(correct_after_CDSS)
    print("Correct CDSS:    {} / {} = {}".format(sum(correct_after_CDSS), len(correct_after_CDSS), rate))

    if kwargs.get("plot"):
        print("plotting...")
        mva = 3
        split_date = datetime.date(2015, 7, 1)
        prefix = kwargs.get("fname_prefix")
        prefix = prefix if prefix is not None else ""

        fname = prefix + "Medication Rate All"
        title = "Medication rate per cardiologist over time"
        plot_data(data, split_date=split_date, mva=mva, fname=fname, title=title, legend=None)

        grouped_data = group_data(data, cdss_practitioners, "CDSS", "No CDSS")

        fname = prefix + "Medication Rate Grouped"
        title = "Medication rate over time grouped by use of CDSS"
        plot_data(grouped_data, split_date=split_date, mva=mva, fname=fname, title=title, legend=grouped_data.keys())
        fname = prefix + "Medication Rate Grouped Complete"
        plot_medication_breakdown(grouped_data, split_date=split_date, mva=mva, fname=fname, title=title,
                                  legend=grouped_data.keys())

        fname = prefix + "Medication Rate Difference"
        title = "Difference in medication rate over time grouped by score"
        plot_difference(grouped_data, split_date=split_date, mva=mva, fname=fname, title=title)

    return data