"""
Django settings for metadata_server project.

For more information on this file, see
https://docs.djangoproject.com/en/1.6/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.6/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
import dotenv
BASE_DIR = os.path.dirname(os.path.dirname(__file__))
dotenv.load_dotenv(os.path.join(BASE_DIR, '.env'))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = os.environ['SECRET_KEY']

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = TEMPLATE_DEBUG = os.environ.has_key('DEBUG')

# Elasticsearch vars
ELASTICSEARCH_URL = os.environ.get('ELASTICSEARCH_URL', 'localhost:9200')
ELASTICSEARCH_INDEX = os.environ.get('ELASTICSEARCH_INDEX', 'manifests')

ALLOWED_HOSTS = [x for x in os.environ.get('ALLOWED_HOSTS','').split(";") if x != '']

IIIF = {
    "imageUriBase":     os.environ.get('IMAGE_URI_BASE', 'http://ids.lib.harvard.edu/ids/iiif/'),
    "serviceBase":      os.environ.get('SERVICE_BASE', 'http://ids.lib.harvard.edu/ids/iiif/'),
    "imageUriSuffix":   "/full/full/full/native",
    "imageInfoSuffix":  "/info.json",
    "manifestUriTmpl":  "http://%s/manifests/",
    "profileLevel":     "http://library.stanford.edu/iiif/image-api/1.1/conformance.html#level1"
}

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'manifests',
    'proxy',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'manifests.exception_logging.ExceptionLoggingMiddleware',
)

ROOT_URLCONF = 'metadata_server.urls'

WSGI_APPLICATION = 'metadata_server.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.6/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.6/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.6/howto/static-files/
STATIC_ROOT = os.path.join(BASE_DIR,'static/')
STATIC_URL = '/static/'

STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
)

TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
)

#logging setup
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'handlers': {
        'file': {
            'level': 'DEBUG',
            'class': 'logging.FileHandler',
            'filename': os.path.join(BASE_DIR, 'debug.log'),
        },
    },
    'loggers': {
        'manifests': {
            'handlers': ['file'],
            'level': 'DEBUG',
            'propagate': True,
        },
    },
}


import os
import sys

import argparse
import string
import numpy as np
from PIL import Image, ImageDraw, ImageFont
from bs4 import BeautifulSoup, NavigableString
import editdistance
import random

import torch
from torch import nn
from torch.backends import cudnn
import torch.nn.functional as F
from torch.utils.data import DataLoader

from dataset import AlignCollate, PILDataset
from model import Model
from utils import CTCLabelConverter, AttnLabelConverter

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
IMG_EXT = {'.jpg', '.tif', '.tiff', '.png'}

def get_already_done(done_path):
    done = set()
    if not os.path.isfile(done_path):
        return done
    with open(done_path, 'r', encoding='utf-8') as fp:
        for line in fp:
            done.add(line.rstrip())
    return done

def get_count(count_path):
    if not os.path.isfile(count_path):
        return 0
    else:
        return int(open(count_path, 'r', encoding='utf-8').readline().rstrip())

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--done_path', type=str, default='result/done.txt')
    parser.add_argument('--count_path', type=str, default='result/count.txt')
    parser.add_argument('--todo_file', type=str, default=None)
    parser.add_argument('--type', type=str, required=True, choices=['dingxiu', 'diaolong'])
    parser.add_argument('--input_path', type=str, default=None)
    parser.add_argument('--output_path', type=str, required=True)
    parser.add_argument('--shuffle', action='store_true', help='random shuffle input list')
    parser.add_argument('--strict', action='store_true', help='use strict check(len diff less than 2 and edit_dist less than 0.2).')
    parser.add_argument('--hold_acc_prob', type=float, default=0.33, help='If gt and pred is the same, hold it with this prob.')

    parser.add_argument('--workers', type=int, help='number of data loading workers', default=4)
    parser.add_argument('--batch_size', type=int, default=192, help='input batch size')
    parser.add_argument('--saved_model', required=True, help="path to saved_model to evaluation")
    """ Data processing """
    parser.add_argument('--batch_max_length', type=int, default=25, help='maximum-label-length')
    parser.add_argument('--imgH', type=int, default=32, help='the height of the input image')
    parser.add_argument('--imgW', type=int, default=100, help='the width of the input image')
    parser.add_argument('--rgb', action='store_true', help='use rgb input')
    parser.add_argument('--character', type=str, default='0123456789abcdefghijklmnopqrstuvwxyz', help='character label')
    parser.add_argument('--sensitive', action='store_true', help='for sensitive character mode')
    parser.add_argument('--PAD', action='store_true', help='whether to keep ratio then pad for image resize')
    """ Model Architecture """
    parser.add_argument('--Transformation', type=str, required=True, help='Transformation stage. None|TPS')
    parser.add_argument('--FeatureExtraction', type=str, required=True, help='FeatureExtraction stage. VGG|RCNN|ResNet')
    parser.add_argument('--SequenceModeling', type=str, required=True, help='SequenceModeling stage. None|BiLSTM')
    parser.add_argument('--Prediction', type=str, required=True, help='Prediction stage. CTC|Attn')
    parser.add_argument('--num_fiducial', type=int, default=20, help='number of fiducial points of TPS-STN')
    parser.add_argument('--input_channel', type=int, default=1, help='the number of input channel of Feature extractor')
    parser.add_argument('--output_channel', type=int, default=512,
                        help='the number of output channel of Feature extractor')
    parser.add_argument('--hidden_size', type=int, default=256, help='the size of the LSTM hidden state')

    parser.add_argument('--page_orient', type=str, choices=['horizontal', 'vertical', 'single'], default='horizontal',
                        help='page orientation, or single char')

    args = parser.parse_args()
    return args

def get_box(db_path):
    boxes = list()
    with open(db_path, 'r', encoding='utf-8') as fp:
        for line in fp:
            line = line.strip().split(',')
            if len(line) % 2 == 1:
                line = line[:-1]
            line = list(map(int, line))
            l = min(line[::2])
            r = max(line[::2])
            u = min(line[1::2])
            d = max(line[1::2])
            boxes.append((l, u, r, d))
    return boxes

def get_gt(html_path, type):
    gt = []
    if type == 'dingxiu':
        with open(html_path, 'r', encoding='utf-8') as fp:
            try:
                html_file = fp.read()
            except UnicodeDecodeError:
                return gt

        soup = BeautifulSoup(html_file, 'lxml')
        lines = soup.find_all('div', {'class': 'linespan'})
        for line in lines:
            contents = line.contents
            cur_line = ''
            br_cnt = 0
            for content in contents:
                if isinstance(content, NavigableString):
                    cur_line += content
                    br_cnt = 0
                else:
                    br_cnt += 1
                    if br_cnt > 1 and len(cur_line) > 0:
                        gt.append(cur_line)
                        cur_line = ''
            if len(cur_line) > 0:
                gt.append(cur_line)
                cur_line = ''
    elif type == 'diaolong':
        pass
    else:
        raise ValueError
    return gt

def demo(args, PIL_image_list, model, AlignCollate_demo, converter):
    dataset = PILDataset(args, PIL_image_list)
    demo_loader = DataLoader(dataset, batch_size=args.batch_size, shuffle=False,
                             num_workers=int(args.workers), collate_fn=AlignCollate_demo,
                             pin_memory=True)
    total_preds_str = []
    with torch.no_grad():
        for image_tensors, image_path_list in demo_loader:
            batch_size = image_tensors.size(0)
            image = image_tensors.to(device)
            # For max length prediction
            length_for_pred = torch.IntTensor([args.batch_max_length] * batch_size).to(device)
            text_for_pred = torch.LongTensor(batch_size, args.batch_max_length + 1).fill_(0).to(device)

            if 'CTC' in args.Prediction:
                preds = model(image, text_for_pred)
                # Select max probabilty (greedy decoding) then decode index to character
                preds_size = torch.IntTensor([preds.size(1)] * batch_size)
                _, preds_index = preds.max(dim=2)
                # preds_index = preds_index.view(-1)
                preds_str = converter.decode(preds_index, preds_size)
            else:
                preds, alphas = model(image, text_for_pred, is_train=False)
                _, preds_index = preds.max(dim=2)
                preds_str = converter.decode(preds_index, length_for_pred)

            for pred in preds_str:
                if 'Attn' in args.Prediction:
                    pred_EOS = pred.find('[s]')
                    pred = pred[:pred_EOS]  # prune after "end of sentence" token ([s])
                total_preds_str.append(pred)
    return total_preds_str

def check_relax(gt, pred):
    if abs(len(gt) - len(pred)) >= 2:
        return 0
    dis = editdistance.distance(gt, pred)
    if dis == 0:
        return 2
    elif dis < max(len(gt), len(pred)) * 0.5:
        return 1
    else:
        return 0

def check_strict(gt, pred):
    if abs(len(gt) - len(pred)) >= 2:
        return 0
    dis = editdistance.distance(gt, pred)
    if dis == 0:
        return 2
    elif dis < max(len(gt), len(pred)) * 0.3:
        return 1
    else:
        return 0

def get_match_idx(preds, gts, strict=False):
    if strict:
        check = check_strict
    else:
        check = check_relax
    matching_idxs = []
    preds_len = len(preds)
    gts_len = len(gts)
    current_pred_idx = 0
    for current_gt_idx in range(gts_len):
        flag = 0
        current_gt = gts[current_gt_idx]
        while True:
            current_pred = preds[current_pred_idx]
            check_res = check(current_gt, current_pred)
            if check_res > 0:
                matching_idxs.append((current_pred_idx, current_gt_idx, check_res))
                break
            else:
                current_pred_idx += 1
                flag += 1
            # max forward match distance = 7
            # if match fail, go back 3
            if flag == 7 or current_pred_idx == preds_len:
                current_pred_idx -= 10
                current_pred_idx = max(0, current_pred_idx)
                break
    return matching_idxs

def get_match_img(args, model, AlignCollate_demo, converter,
                  img_path, db_path, html_path, current_cnt):
    match_cnt = 0
    acc_cnt = 0
    should_cnt = 0
    boxes = get_box(db_path)
    gts = get_gt(html_path, args.type)
    PIL_image_list = list()
    img = Image.open(img_path).convert('L')
    for box in boxes:
        crop_img = img.crop(box)
        # crop_img.show()
        PIL_image_list.append(crop_img)
    pred = demo(args, PIL_image_list, model, AlignCollate_demo, converter)
    matching_idx = get_match_idx(pred, gts, args.strict)
    for p_idx, g_idx, check_res in matching_idx:
        if check_res == 1:
            img = PIL_image_list[p_idx]
            img_gt = gts[g_idx]
            img_save_path = os.path.join(args.output_path, 'imgs', str(current_cnt) + '.jpg')
            img.save(img_save_path)
            gt_save_path = os.path.join(args.output_path, 'gts', str(current_cnt) + '.txt')
            with open(gt_save_path, 'w', encoding='utf-8') as fp:
                fp.write(img_gt + '\n')
            current_cnt += 1
            match_cnt += 1
            should_cnt += 1
        elif check_res == 2:
            if random.random() < args.hold_acc_prob:
                img = PIL_image_list[p_idx]
                img_gt = gts[g_idx]
                img_save_path = os.path.join(args.output_path, 'imgs', str(current_cnt) + '.jpg')
                img.save(img_save_path)
                gt_save_path = os.path.join(args.output_path, 'gts', str(current_cnt) + '.txt')
                with open(gt_save_path, 'w', encoding='utf-8') as fp:
                    fp.write(img_gt + '\n')
                current_cnt += 1
                match_cnt += 1
            should_cnt += 1
            acc_cnt += 1
        else:
            raise ValueError('check_res should be 1 or 2')
    return match_cnt, acc_cnt, should_cnt, current_cnt

def init_model(args):
    """ model configuration """
    if 'CTC' in args.Prediction:
        converter = CTCLabelConverter(args.character)
    else:
        converter = AttnLabelConverter(args.character)
    args.num_class = len(converter.character)

    if args.rgb:
        args.input_channel = 3
    model = Model(args)
    print('model input parameters', args.imgH, args.imgW, args.num_fiducial, args.input_channel, args.output_channel,
          args.hidden_size, args.num_class, args.batch_max_length, args.Transformation, args.FeatureExtraction,
          args.SequenceModeling, args.Prediction)
    try:
        model = torch.nn.DataParallel(model).to(device)
    except RuntimeError:
        raise RuntimeError(device)

    # load model
    print('loading pretrained model from %s' % args.saved_model)
    model.load_state_dict(torch.load(args.saved_model, map_location=device))

    # prepare data. two demo images from https://github.com/bgshih/crnn#run-demo
    AlignCollate_demo = AlignCollate(imgH=args.imgH, imgW=args.imgW, keep_ratio_with_pad=args.PAD)
    model.eval()
    return converter, model, AlignCollate_demo

if __name__ == '__main__':
    args = parse_args()
    done = get_already_done(args.done_path)
    print('{} imgs already done.'.format(len(done)))
    cnt = get_count(args.count_path)
    print('{} lines has been extracted.'.format(cnt))
    if args.todo_file is None:
        print('no todo_file in args. Scan input_path.')
        todo_list = list()
        for root, dirs, files in os.walk(args.input_path):
            for file in files:
                if os.path.splitext(file)[-1].lower() not in IMG_EXT:
                    continue
                file_path = os.path.join(root, file)
                if file_path in done:
                    continue
                todo_list.append(file_path)
    else:
        print('load todo_file from {}.'.format(args.todo_file))
        todo_list = [s.rstrip() for s in open(args.todo_file, 'r', encoding='utf-8').readlines()]
        todo_list = [s for s in todo_list if s not in done]
    print('{} files to do.'.format(len(todo_list)), flush=True)
    if args.shuffle:
        print('shuffle todo files.')
        random.shuffle(todo_list)
    os.makedirs(os.path.join(args.output_path, 'imgs'), exist_ok=True)
    os.makedirs(os.path.join(args.output_path, 'gts'), exist_ok=True)

    if args.character in ['CN-s', 'CN-m', 'CN-l', 'CN-xl']:
        size = args.character.split('-')[-1]
        with open('charset/charset_' + size + '.txt', 'r', encoding='utf-8') as chars:
            charset = [c.strip() for c in chars]
        charset = ''.join(charset)
        args.character = charset
    elif args.sensitive:
        # opt.character += 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
        args.character = string.printable[:-6]  # same with ASTER setting (use 94 char).
    else:
        raise ValueError

    cudnn.benchmark = True
    cudnn.deterministic = True
    args.num_gpu = torch.cuda.device_count()

    converter, model, AlignCollate_demo = init_model(args)

    for file_path in todo_list:
        db_result_path = os.path.splitext(file_path)[0] + '.txt'
        if not os.path.isfile(db_result_path):
            continue
        html_gt_path = os.path.splitext(file_path)[0] + '.html'
        print('started from No. {} '.format(cnt), end='')
        match_cnt, acc_cnt, should_cnt, cnt = get_match_img(
            args, model, AlignCollate_demo, converter, file_path, db_result_path, html_gt_path, cnt
        )
        print('{} should get {} imgs, get {} match imgs, {} imgs perfectly right.'.format(
            file_path, should_cnt, match_cnt, acc_cnt)
        )
        with open(args.done_path, 'a', encoding='utf-8') as fp:
            fp.write(file_path + '\n')
        with open(args.count_path, 'w', encoding='utf-8') as fp:
            fp.write(str(cnt) + '\n')


# -*- coding: utf-8 -*-
from django.conf.urls import include, url
from django.contrib import admin
import view
from catalog import plup

urlpatterns = [
    url(r'^$', view.main),
    url(r'^products_and_services/$', view.prod_and_serv),
    url(r'^contact/$', view.contact),
    url(r'^admin/', include(admin.site.urls)),
    url(r'^tinymce/', include('tinymce.urls')),
    url(r'^products_and_services/(\d+)/$',view.item_desc),
    # url(r'^plupload/$', plup.plupload_page),
    url(r'^plup_service/$', plup.plupload_service),
    url(r'^imagelibrary/$',view.get_media),
    url(r'^products/$', view.products),
    url(r'^alphabet/$', view.alphabet),
    url(r'^alphabet/index/$', view.alpha_index),
    url(r'^del_media/$', plup.del_media),
    url(r'^upload_file/$',view.upload_file),

]

''' выдача медиа файлов на дев сервере, надо будет выпилить'''
from django.conf.urls.static import static
from django.contrib.staticfiles.urls import staticfiles_urlpatterns
from electrofarfor import settings

urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
urlpatterns += staticfiles_urlpatterns()


import tweepy

from authorization_tokens import *

# import random
random_number = 3

if random_number == 0:
        #option1
        #Pick a phrase randomly from a list of phrases:
        phrase_list = ["Hi my name is Kasper",
                        "Kasper is my name",
                        "I like Twitter Bots"]
        random_number = random.randrange( len(phrase_list))
        message = phrase_list[random_number]
elif random_number == 1:
        #option 2
        # Create a sentence template with some blanks, and
        #randomly pick a word from a list to fill in each blank.
        word_list = ["apples", "bananas", "carrots", "dates"]

        string_template = "Some people think {} are good, but I like {}."

        random_number = random.randrange( len(word_list))
        word1 = word_list[random_number]

        random_number = random.randrange( len(word_list))
        word2 = word_list[random_number]

        message = string_template.format(word1, word2)
elif random_number == 2:
        #option 3
        #Randomly pick a template from a list, then randomly pick word_list
        #from a word list, and use the words to fill in the string_template

        word_list = ["people", "video games", "sports", "people"]
        template_list = ["If you like {}, you'll love {}.",
                        "You might think I'm a {} person, but actually I love {}.",
                        "You'd never guess it, but I like {} even more than {}."]

        random_number = random.randrange( len(template_list))
        template = template_list[random_number]

        random_number = random.randrange( len(word_list))
        word1 = word_list[random_number]

        random_number = random.randrange( len(word_list))
        word2 = word_list[random_number]

        message = template.format(word1, word2)
elif random_number == 3:
    # Kasper Bielecki 09/12/2020
    # Dungeons and Dragans Charachter Generator
    # Based on a Java Program I worked on between the dates of 04/17/2019 - 09/12/2020
    #*************CHECKLIST*************
    #
    # - Physical Attributes -
    # [*] Race
    # [*] Sub Race
    # [*] Sex
    # [*] Height
    # [*] Weight
    # [*] Age and agebracket
    # [] Eyes
    # [] Skin
    # [] Hair
    #
    # - Character Traits -
    # [*] Alignment
    # [*] Class
    # [] Background
    # [] Proficiencies
    # [] Character Name
    # [] Skills
    # [] Languages
    # [] Personality Traits
    # [] Ideals
    # [] Bonds
    # [] Flaws
    #
    # - Statistics -
    # [] Level
    # [*] Abilities
    # [*] Modifiers
    # [] Saving Throws
    # [] Hit Points
    # [] Hit Dice
    # [] Perception
    # [] Spell Casting Ability
    # [] Spell Save DC
    # [] Spell Attack Bonus
    # [] Spell Casting Class
    #
    # - Others -
    # [] Attacks
    # [] Equipment
    # [] Spells
    #
    #*******************************
    import random

    #04 Sided Die
    def D4():
    	x = (random.randrange(1,5))
    	return x

    #06 Sided Die
    def D6():
    	x = (random.randrange(1,7))
    	return x

    #08 Sided Die
    def D8():
    	x = (random.randrange(1,9))
    	return x

    #10 Sided Die
    def D10():
    	x = (random.randrange(1,11))
    	return x

    #Ability Score Roll
    def ability_score_roll():
        roll1 = D6()
        roll2 = D6()
        roll3 = D6()
        roll4 = D6()

        rolls = [
        roll1,
        roll2,
        roll3,
        roll4
        ]

        rolls.sort()

        ability_score = rolls[1] + rolls[2] + rolls[3]
        return ability_score

    def modifier_(ability):
        if ability == 1:
            return "(-5)"
        elif ability == 2 or ability == 3:
            return "(-4)"
        elif ability == 4 or ability == 5:
            return "(-3)"
        elif ability == 6 or ability == 7:
            return "(-2)"
        elif ability == 8 or ability == 9:
            return "(-1)"
        elif ability == 10 or ability == 11:
            return "(+0)"
        elif ability == 12 or ability == 13:
            return "(+1)"
        elif ability == 14 or ability == 15:
            return "(+2)"
        elif ability == 16 or ability == 17:
            return "(+3)"
        elif ability == 18 or ability == 19:
            return "(+4)"
        elif ability == 20 or ability == 21:
            return "(+5)"
        elif ability == 22 or ability == 23:
            return "(+6)"
        elif ability == 24 or ability == 25:
            return "(+7)"
        elif abilit == 26 or ability == 27:
            return "(+8)"
        elif ability == 28 or ability == 29:
            return "(+9)"
        elif ability == 30:
             return "(+10)"

    #Race
    race_list = [
    "Dragonborn",
    "Dwarf",
    "Elf",
    "Gnome",
    "Half-Elf",
    "Halfling",
    "Half-Orc",
    "Human",
    "Tiefling"
    ]
    random_number = random.randrange( len(race_list))
    race = race_list[random_number]

    #Sub Race
    if race == "Dwarf":
        subrace_list = [
        "Hill",
        "Mountain"
        ]
        random_number = random.randrange( len(subrace_list))
        subrace = subrace_list[random_number]

    elif race == "Elf":
        subrace_list = [
        "High",
        "Wood",
        "Drow"
        ]
        random_number = random.randrange( len(subrace_list))
        subrace = subrace_list[random_number]

    elif race == "Halfling":
        subrace_list = [
        "Lightfoot",
        "Stout"
        ]
        random_number = random.randrange( len(subrace_list))
        subrace = subrace_list[random_number]

    elif race == "Human":
        subrace_list = [
        "Calishite",
        "Chondathan",
        "Damaran",
        "Illuskan",
        "Mulan",
        "Rashemi",
        "Shou",
        "Tethyrian",
        "Turami"
        ]
        random_number = random.randrange( len(subrace_list))
        subrace = subrace_list[random_number]

    elif race == "Dragonborn":
        subrace_list = [
        "Black",
        "Blue",
        "Brass",
        "Bronze",
        "Copper",
        "Gold",
        "Green",
        "Red",
        "Silver",
        "White"
        ]
        random_number = random.randrange( len(subrace_list))
        subrace = subrace_list[random_number]

    elif race == "Gnome":
        subrace_list = [
        "Forrest",
        "Rock"
        ]
        random_number = random.randrange( len(subrace_list))
        subrace = subrace_list[random_number]

    else:
        subrace = None

    #Class
    class_list = [
    "Barbarian",
    "Bard",
    "Cleric",
    "Druid",
    "Fighter",
    "Monk",
    "Paladin",
    "Ranger",
    "Rogue"
    ]
    random_number = random.randrange( len(class_list))
    class_ = class_list[random_number]

    #Sex
    sex_list = [
    "Male",
    "Female"
    ]
    random_number = random.randrange( len(sex_list))
    sex = sex_list[random_number]

    #Height and Weight
    if race == "Dragonborn":
        roll = D8() + D8()
        feet = int((66 + roll) / 12)
        inches = (66 + roll) % 12
        weight = ((roll * (D6()+D6())) + 175)

    elif race == "Dwarf":
        if subrace == "Hill":
            roll = D4() + D4()
            feet = int((44 + roll) / 12)
            inches = (44 + roll) % 12
            weight = ((roll * (D6()+D6())) + 115)

        elif subrace == "Mountain":
            roll = D4() + D4()
            feet = int((48 + roll) / 12)
            inches = (48 + roll) % 12
            weight = ((roll * (D6()+D6())) + 130)

    elif race == "Elf":
        if subrace == "High":
            roll = D10() + D10()
            feet = int((54 + roll) / 12)
            inches = (54 + roll) % 12
            weight = ((roll * D4()) + 90)

        elif subrace == "Wood":
            roll = D10() + D10()
            feet = int((54 + roll) / 12)
            inches = (54 + roll) % 12
            weight = ((roll * D4()) + 100)

        elif subrace == "Drow":
            roll = D6() + D6()
            feet = int((53 + roll) / 12)
            inches = (53 + roll) % 12
            weight = ((roll * D6()) + 75)

    elif race == "Gnome":
        roll = D4() + D4()
        feet = int((35 + roll) / 12)
        inches = (35 + roll) % 12
        weight = ((roll * 1) + 35)

    elif race == "Half-Elf":
        roll = D8() + D8()
        feet = int((57 + roll) / 12)
        inches = (57 + roll) % 12
        weight = ((roll * (D4()+D4())) + 110)

    elif race == "Halfling":
        roll = D4() + D4()
        feet = int((31 + roll) / 12)
        inches = (31 + roll) % 12
        weight = ((roll * 1) + 35)

    elif race == "Half-Orc":
        roll = D10() + D10()
        feet = int((58 + roll) / 12)
        inches = (58 + roll) % 12
        weight = ((roll * (D6()+D6())) + 140)

    elif race == "Human":
        roll = D10() + D10()
        feet = int((56 + roll) / 12)
        inches = (56 + roll) % 12
        weight = ((roll * (D4()+D4())) + 110)

    elif race == "Tiefling":
        roll = D8() + D8()
        feet = int((57 + roll) / 12)
        inches = (57 + roll) % 12
        weight = ((roll * (D4()+D4())) + 110)

    height_template = '''{}'{}"'''
    height, weight = (height_template.format(feet, inches), weight)

    #Age and Age Bracket
    if race == "Human":
        age = (random.randrange(6,91))
        if age < 18:
            age_bracket = "years old - Childhood"
        elif age >= 18 and age < 30:
            age_bracket = "years old - Young Adult"
        elif age >= 30 and age < 65:
            age_bracket = "years old - Settled Adult"
        elif age >= 65 and age <= 90:
            age_bracket = "years old - Elderly"

    elif race == "Dwarf":
        age = (random.randrange(6,351))
        if age < 18:
            age_bracket = "years old - Childhood"
        elif age >= 18 and age < 50:
            age_bracket = "years old - Young Adult"
        elif age >= 50 and age < 200:
            age_bracket = "years old - Settled Adult"
        elif age >= 200 and age <= 350:
            age_bracket = "years old - Elderly"

    elif race == "Elf":
        age = (random.randrange(6,751))
        if age < 100:
            age_bracket = "years old - Childhood"
        elif age >= 100 and age < 450:
            age_bracket = "years old - Young Adult"
        elif age >= 450 and age < 650:
            age_bracket = "years old - Settled Adult"
        elif age >= 650 and age <= 750:
            age_bracket = "years old - Elderly"

    elif race == "Halfling":
        age = (random.randrange(6,251))
        if age < 20:
            age_bracket = "years old - Childhood"
        elif age >= 20 and age < 40:
            age_bracket = "years old - Young Adult"
        elif age >= 40 and age < 150:
            age_bracket = "years old - Settled Adult"
        elif age >= 150 and age <= 250:
            age_bracket = "years old - Elderly"

    elif race == "Dragonborn":
        age = (random.randrange(1,81))
        if age < 15:
            age_bracket = "years old - Childhood"
        elif age >= 15 and age < 20:
            age_bracket = "years old - Young Adult"
        elif age >= 20 and age < 50:
            age_bracket = "years old - Settled Adult"
        elif age >= 50 and age <= 80:
            age_bracket = "years old - Elderly"

    elif race == "Gnome":
        age = (random.randrange(6,401))
        if age < 20:
            age_bracket = "years old - Childhood"
        elif age >= 20 and age < 40:
            age_bracket = "years old - Young Adult"
        elif age >= 40 and age < 200:
            age_bracket = "years old - Settled Adult"
        elif age >= 200 and age <= 400:
            age_bracket = "years old - Elderly"

    elif race == "Half-Elf":
        age = (random.randrange(6,181))
        if age < 20:
            age_bracket = "years old - Childhood"
        elif age >= 20 and age < 50:
            age_bracket = "years old - Young Adult"
        elif age >= 50 and age < 150:
            age_bracket = "years old - Settled Adult"
        elif age >= 150 and age <= 180:
            age_bracket = "years old - Elderly"

    elif race == "Half-Orc":
        age = (random.randrange(1,76))
        if age < 14:
            age_bracket = "years old - Childhood"
        elif age >= 14 and age < 45:
            age_bracket = "years old - Young Adult"
        elif age >= 45 and age < 60:
            age_bracket = "years old - Settled Adult"
        elif age >= 60 and age <= 75:
            age_bracket = "years old - Elderly"

    elif race == "Tiefling":
        age = (random.randrange(6,101))
        if age < 18:
            age_bracket = "years old - Childhood"
        elif age >= 18 and age < 40:
            age_bracket = "years old - Young Adult"
        elif age >= 40 and age < 70:
            age_bracket = "years old - Settled Adult"
        elif age >= 70 and age <= 100:
            age_bracket = "years old - Elderly"

    #Alignment
    alignment_list = [
    "Lawful Good",
    "Neutral Good",
    "Chaotic Good",
    "Lawful Neutral",
    "True Neutral",
    "Chaotic Neutral",
    "Lawful Evil",
    "Neutral Evil",
    "Chaotic Evil"
    ]
    random_number = random.randrange( len(alignment_list))
    alignment = alignment_list[random_number]

    #Abilities
    ability1 = ability_score_roll()
    ability2 = ability_score_roll()
    ability3 = ability_score_roll()
    ability4 = ability_score_roll()
    ability5 = ability_score_roll()
    ability6 = ability_score_roll()

    abilities = [ability1,ability2,ability3,ability4,ability5,ability6]
    abilities.sort()

    if class_ == "Barbarian":
        strength = abilities[4]
        dexterity = abilities[3]
        constitution = abilities[5]
        intelligence = abilities[0]
        wisdom = abilities[1]
        charisma = abilities[2]
    elif class_ == "Bard":
        strength = abilities[1]
        dexterity = abilities[4]
        constitution = abilities[3]
        intelligence = abilities[0]
        wisdom = abilities[2]
        charisma = abilities[5]
    elif class_ == "Cleric":
        strength = abilities[4]
        dexterity = abilities[1]
        constitution = abilities[3]
        intelligence = abilities[0]
        wisdom = abilities[5]
        charisma = abilities[2]
    elif class_ == "Druid":
        strength = abilities[0]
        dexterity = abilities[4]
        constitution = abilities[3]
        intelligence = abilities[1]
        wisdom = abilities[5]
        charisma = abilities[2]
    elif class_ == "Fighter":
        strength = abilities[5]
        dexterity = abilities[3]
        constitution = abilities[4]
        intelligence = abilities[0]
        wisdom = abilities[1]
        charisma = abilities[2]
    elif class_ == "Monk":
        strength = abilities[0]
        dexterity = abilities[5]
        constitution = abilities[3]
        intelligence = abilities[1]
        wisdom = abilities[4]
        charisma = abilities[2]
    elif class_ == "Paladin":
        strength = abilities[5]
        dexterity = abilities[2]
        constitution = abilities[4]
        intelligence = abilities[0]
        wisdom = abilities[1]
        charisma = abilities[3]
    elif class_ == "Ranger":
        strength = abilities[0]
        dexterity = abilities[5]
        constitution = abilities[3]
        intelligence = abilities[1]
        wisdom = abilities[4]
        charisma = abilities[2]
    elif class_ == "Rogue":
        strength = abilities[3]
        dexterity = abilities[5]
        constitution = abilities[2]
        intelligence = abilities[0]
        wisdom = abilities[1]
        charisma = abilities[4]
    elif class_ == "Sorcerer":
        strength = abilities[0]
        dexterity = abilities[4]
        constitution = abilities[3]
        intelligence = abilities[1]
        wisdom = abilities[2]
        charisma = abilities[5]
    elif class_ == "Warlock":
        strength = abilities[1]
        dexterity = abilities[3]
        constitution = abilities[4]
        intelligence = abilities[0]
        wisdom = abilities[2]
        charisma = abilities[5]
    elif class_ == "Wizard":
        strength = abilities[0]
        dexterity = abilities[3]
        constitution = abilities[4]
        intelligence = abilities[5]
        wisdom = abilities[2]
        charisma = abilities[1]

    if race == "Dwarf":
        constitution = constitution + 2
        if subrace == "Hill":
            wisdom = wisdom + 1
        elif subrace == "Mountain":
            strength = strength + 2
    elif race == "Elf":
        dexterity = dexterity + 2
        if subrace == "High":
            intelligence = intelligence + 1
        elif subrace == "Drow":
            charisma = charisma + 1
        elif subrace == "Wood":
            wisdow = wisdom + 1
    elif race == "Halfling":
        dexterity = dexterity + 2
        if subrace == "Lightfoot":
            charisma = charisma + 1
        elif subrace == "Stout":
            constitution = constitution + 1
    elif race == "Human":
        strength = strength + 1
        dexterity = dexterity + 1
        constitution = constitution + 1
        intelligence = intelligence + 1
        wisdom = wisdom + 1
        charisma = charisma + 1
    elif race == "Dragonborn":
        strength = strength + 2
        charisma = charisma + 1
    elif race == "Gnome":
        intelligence = intelligence + 2
        if subrace == "Forrest":
            dexterity = dexterity + 1
        elif subrace == "Rock":
            constitution = constitution + 1
    elif race == "Half-Elf":
        charisma = charisma + 2
        abilitiy_choices = ["strength","dexterity","constitution","intelligence","wisdom","charisma"]
        random_number = random.randrange( len(abilitiy_choices))
        if abilitiy_choices[random_number] == "strength":
            strength = strength +1
            abilities.pop(0)
            random_number = random.randrange( len(abilitiy_choices))
            if abilitiy_choices[random_number] == "dexterity":
                dexterity = dexterity + 1
            elif abilitiy_choices[random_number] == "constitution":
                constitution = constitution + 1
            elif abilitiy_choices[random_number] == "intelligence":
                intelligence = intelligence + 1
            elif abilitiy_choices[random_number] == "wisdom":
                wisdom = wisdom + 1
            elif abilitiy_choices[random_number] == "charisma":
                charisma = charisma + 1
        elif abilitiy_choices[random_number] == "dexterity":
            dexterity = dexterity +1
            abilities.pop(1)
            random_number = random.randrange( len(abilitiy_choices))
            if abilitiy_choices[random_number] == "strength":
                strength = strength + 1
            elif abilitiy_choices[random_number] == "constitution":
                constitution = constitution + 1
            elif abilitiy_choices[random_number] == "intelligence":
                intelligence = intelligence + 1
            elif abilitiy_choices[random_number] == "wisdom":
                wisdom = wisdom + 1
            elif abilitiy_choices[random_number] == "charisma":
                charisma = charisma + 1
        elif abilitiy_choices[random_number] == "constitution":
            constitution = constitution +1
            abilities.pop(2)
            random_number = random.randrange( len(abilitiy_choices))
            if abilitiy_choices[random_number] == "strength":
                strength = strength + 1
            elif abilitiy_choices[random_number] == "dexterity":
                dexterity = dexterity + 1
            elif abilitiy_choices[random_number] == "intelligence":
                intelligence = intelligence + 1
            elif abilitiy_choices[random_number] == "wisdom":
                wisdom = wisdom + 1
            elif abilitiy_choices[random_number] == "charisma":
                charisma = charisma + 1
        elif abilitiy_choices[random_number] == "intelligence":
            intelligence = intelligence +1
            abilities.pop(3)
            random_number = random.randrange( len(abilitiy_choices))
            if abilitiy_choices[random_number] == "strength":
                strength = strength + 1
            elif abilitiy_choices[random_number] == "dexterity":
                dexterity = dexterity + 1
            elif abilitiy_choices[random_number] == "constitution":
                constitution = constitution + 1
            elif abilitiy_choices[random_number] == "wisdom":
                wisdom = wisdom + 1
            elif abilitiy_choices[random_number] == "charisma":
                charisma = charisma + 1
        elif abilitiy_choices[random_number] == "wisdom":
            wisdom = wisdom +1
            abilities.pop(4)
            random_number = random.randrange( len(abilitiy_choices))
            if abilitiy_choices[random_number] == "strength":
                strength = strength + 1
            elif abilitiy_choices[random_number] == "dexterity":
                dexterity = dexterity + 1
            elif abilitiy_choices[random_number] == "constitution":
                constitution = constitution + 1
            elif abilitiy_choices[random_number] == "intelligence":
                intelligence = intelligence + 1
            elif abilitiy_choices[random_number] == "charisma":
                charisma = charisma + 1
        elif abilitiy_choices[random_number] == "charisma":
            charisma = charisma +1
            abilities.pop(5)
            random_number = random.randrange( len(abilitiy_choices))
            if abilitiy_choices[random_number] == "strength":
                strength = strength + 1
            elif abilitiy_choices[random_number] == "dexterity":
                dexterity = dexterity + 1
            elif abilitiy_choices[random_number] == "constitution":
                constitution = constitution + 1
            elif abilitiy_choices[random_number] == "intelligence":
                intelligence = intelligence + 1
            elif abilitiy_choices[random_number] == "wisdom":
                wisdom = wisdom + 1
    elif race == "Half-Orc":
        strength = strength + 2
        constitution = constitution + 1
    elif race == "Tiefling":
        intelligence = intelligence +1
        charisma = charisma + 2

    strength_modifier = modifier_(strength)
    dexterity_modifier = modifier_(dexterity)
    constitution_modifier = modifier_(constitution)
    intelligence_modifier = modifier_(intelligence)
    wisdom_modifier = modifier_(wisdom)
    charisma_modifier = modifier_(charisma)
    #Print
    if subrace == None:
        # print("Race:",race,
        # "\nClass:",class_,
        # "\nAlignment:",alignment,
        # "\nSex:",sex,
        # "\nHeight:",height,
    	# "\nWeight:",weight,"lbs",
        # "\nAge:",age,age_bracket,
        # "\nStrength:",strength,strength_modifier,
        # "\nDexterity:",dexterity,dexterity_modifier,
        # "\nConstitution:",constitution,constitution_modifier,
        # "\nIntelligence:",intelligence,intelligence_modifier,
        # "\nWisdom:",wisdom,wisdom_modifier,
        # "\nCharisma:",charisma,charisma_modifier,)

        template = "Race: {}\nClass: {}\nAlignment: {}\nSex: {}\nHeight: {}\nWeight: {} Lbs\nAge: {} {}\nStrength: {} {}\nDexterity: {} {}\nConstitution {} {}\nIntelligence: {} {}\nWisdom: {} {}\nCharisma {} {}"
        message = template.format(race,class_,alignment,sex,height,weight,age,age_bracket,strength,strength_modifier,dexterity,dexterity_modifier,constitution,constitution_modifier,intelligence,intelligence_modifier,wisdom,wisdom_modifier,charisma,charisma_modifier)
    else:
        # print("Race:",subrace,race,
        # "\nClass:",class_,
        # "\nAlignment:",alignment,
        # "\nSex:",sex,
        # "\nHeight:",height,
    	# "\nWeight:",weight,"lbs",
        # "\nAge:",age,age_bracket,
        # "\nStrength:",strength,strength_modifier,
        # "\nDexterity:",dexterity,dexterity_modifier,
        # "\nConstitution:",constitution,constitution_modifier,
        # "\nIntelligence:",intelligence,intelligence_modifier,
        # "\nWisdom:",wisdom,wisdom_modifier,
        # "\nCharisma:",charisma,charisma_modifier,)
        template = "Race: {} {}\nClass: {}\nAlignment: {}\nSex: {}\nHeight: {}\nWeight: {} Lbs\nAge: {} {}\nStrength: {} {}\nDexterity: {} {}\nConstitution {} {}\nIntelligence: {} {}\nWisdom: {} {}\nCharisma {} {}"
        message = template.format(subrace,race,class_,alignment,sex,height,weight,age,age_bracket,strength,strength_modifier,dexterity,dexterity_modifier,constitution,constitution_modifier,intelligence,intelligence_modifier,wisdom,wisdom_modifier,charisma,charisma_modifier)

auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)

api = tweepy.API(auth)

api.update_status(message)
print("Done.")


from gensim.models import word2vec
import MeCab
import sys
import re
from collections import Counter

#出題された単語を０番目、それ以降を候補としてリストで渡された場合を想定している

def WrongWord(Question):

	WrongWord_list = []#誤解等のリストの用意
	Word_list = []#分かちがきした単語のリストの用意
	Word_list2 = {}#辞書型で結果を保存するための用意

	#以下、単語じゃなかった場合などに対処するため、分かち書きの処理
	for i in Question:
		parse = MeCab.Tagger().parse(i)
		lines = parse.split('\n')
		items = (re.split('[\t,]', line) for line in lines)

        #分かちがきしたものをリストに保存する
		for item in items:
			word = item[0]
			Word_list.append(word)
			break

	#リストから出題された問題の単語を取り出す
	original = Word_list.pop(0)

	#以下、word2vecを使い、類似した単語を探す処理
	model = word2vec.Word2Vec.load("./wiki.model")

	#出題された単語と渡された各候補の類似度を計算して辞書型として保存する
	for i in Word_list:
		similarity = model.wv.similarity(original,i)
		Word_list2[i] = similarity

	print(Word_list2)#確認用
	#降順にソートしてリストに保存,類似度上位３つをリストで出力する
	num = 0

	for k, v in sorted(Word_list2.items(), key = lambda x:-x[1]):
		WrongWord_list.append(k)
		num += 1
		if num == 3:
			break

	return WrongWord_list


import logger
import argparse
import os
import json
import numpy as np; np.random.seed(0)

def main(args):
    render = args.render
    if not render:
        import matplotlib
        matplotlib.use('Agg')
        import matplotlib.pyplot as plt
    from envs import DoubleIntegratorEnv, MountainCarEnv, CartPoleEnv, SwingUpEnv
    from utils.utils import TabularPolicy, TabularValueFun
    from part1.tabular_value_iteration import ValueIteration
    from part2.look_ahead_policy import LookAheadPolicy
    from part2.discretize import Discretize
    envs = [DoubleIntegratorEnv(), MountainCarEnv(), CartPoleEnv(), SwingUpEnv()]

    for env in envs:
        env_name = env.__class__.__name__

        if env_name == 'MountainCarEnv':
            state_discretization = 51
        else:
            state_discretization = 21
        exp_dir = os.getcwd() + '/data/part2_d/%s/policy_type%s_mode%s_horizon%s/' % (env_name,
                                                                              args.policy_type,
                                                                              args.mode,
                                                                              args.horizon)
        logger.configure(dir=exp_dir, format_strs=['stdout', 'log', 'csv'])
        args_dict = vars(args)
        args_dict['env'] = env_name
        json.dump(vars(args), open(exp_dir + '/params.json', 'w'), indent=2, sort_keys=True)

        env = Discretize(env,
                         state_discretization=state_discretization,
                         mode=args.mode
                         )
        value_fun = TabularValueFun(env)
        if args.policy_type == 'tabular':
            policy = TabularPolicy(env)
        elif args.policy_type == 'look_ahead':
            policy = LookAheadPolicy(env, value_fun, args.horizon)
        else:
            raise NotImplementedError
        algo = ValueIteration(env,
                              value_fun,
                              policy,
                              render=render,
                              max_itr=args.max_iter,
                              num_rollouts=1,
                              render_itr=5,
                              log_itr=5)
        algo.train()

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--exp_name", "-n", type=str, default='test',
                        help="folder name where the experiment will be saved")
    parser.add_argument("--render", "-r", action='store_true',
                        help="Vizualize the policy and contours when training")
    parser.add_argument("--mode", "-m", type=str, default='linear', choices=['nn', 'linear'],
                        help="Mode of interpolate between discrete points")
    parser.add_argument("--policy_type", "-p", type=str, default='look_ahead', choices=['tabular', 'look_ahead'],
                        help='Type of policy to use. Whether to use look ahead policy or tabular')
    parser.add_argument("--horizon", "-H", type=int, default=1,
                        help='Planning horizon for the look ahead policy')
    parser.add_argument("--max_iter", "-i", type=int, default=150,
                        help='Maximum number of iterations for the value iteration algorithm')
    args = parser.parse_args()
    main(args)


import pandas as pd
from datetime import datetime

df = pd.read_csv('simple_nyc_green.csv')

count_pickup = {}
year_2013 = datetime.strptime('01-01-13', '%m-%d-%y').date().year

for index, row in df.iterrows():
    dt = datetime.strptime(row['pickup_datetime'], '%m-%d-%y %H:%M:%S').date()
    date = dt.strftime("%m-%d-%y")
    # print(dt)
    c = int(row['passenger_count'])
    # print(c)
    if date in count_pickup:
        count_pickup[date] += c
    else:
        count_pickup[date] = c

# print(count_pickup)

data_lst = []
value_lst = []
for date in count_pickup.keys():
    data_lst.append(date)
    value_lst.append(count_pickup[date])

data = {'Date': data_lst, 'Volumn': value_lst }
df = pd.DataFrame(data)

# def ret(row):
#     return count_pickup[row['Date']]
# df['volumn'] = df.apply(ret, axis=1)
df.to_csv('counts.csv')

# def ct(row):
#     # 08-25-13 03:04:20
#     dt = datetime.strptime(row['pickup_datetime'], '%m-%d-%y %H:%M:%S').date()
#     print(dt)
#     c = int(row['passenger_count'])
#     print(c)
#     if dt in count_pickup:
#         count_pickup[dt] += c
#     else:
#         count_pickup[dt] = c

#data = {"":}
# count = df.apply(ct, axis=1)

# df = pd.DataFrame(count)
# df.to_csv('counts.csv')


import torch
import logging
from typing import Optional, List
from dataclasses import dataclass, field
from transformers.file_utils import cached_property, torch_required

logger = logging.getLogger(__name__)

@dataclass
class CHMMArguments:
    """
    Arguments regarding the training of Neural hidden Markov Model
    """
    train_file: Optional[str] = field(
        default='', metadata={'help': 'training data name'}
    )
    valid_file: Optional[str] = field(
        default='', metadata={'help': 'development data name'}
    )
    test_file: Optional[str] = field(
        default='', metadata={'help': 'test data name'}
    )
    output_dir: Optional[str] = field(
        default='.',
        metadata={"help": "The output folder where the model predictions and checkpoints will be written."},
    )
    trans_nn_weight: Optional[float] = field(
        default=1.0, metadata={'help': 'the weight of neural part in the transition matrix'}
    )
    no_neural_emiss: Optional[bool] = field(
        default=False, metadata={'help': 'Not use neural networks to predict emission probabilities.'}
    )
    emiss_nn_weight: Optional[float] = field(
        default=1.0, metadata={'help': 'the weight of neural part in the emission matrix'}
    )
    num_lm_train_epochs: Optional[int] = field(
        default=15, metadata={'help': 'number of denoising model training epochs'}
    )
    num_lm_nn_pretrain_epochs: Optional[int] = field(
        default=5, metadata={'help': 'number of denoising model pre-training epochs'}
    )
    num_lm_valid_tolerance: Optional[int] = field(
        default=10, metadata={"help": "How many tolerance epochs before quiting training"}
    )
    hmm_lr: Optional[float] = field(
        default=0.01, metadata={'help': 'learning rate of the original hidden markov model transition and emission'}
    )
    nn_lr: Optional[float] = field(
        default=0.001, metadata={'help': 'learning rate of the neural networks in CHMM'}
    )
    lm_batch_size: Optional[int] = field(
        default=128, metadata={'help': 'denoising model training batch size'}
    )
    obs_normalization: Optional[bool] = field(
        default=False, metadata={'help': 'whether normalize observations'}
    )
    bert_model_name_or_path: Optional[str] = field(
        default='', metadata={"help": "Path to pretrained BERT model or model identifier from huggingface.co/models; "
                                      "Used to construct BERT embeddings if not exist"}
    )
    no_cuda: Optional[bool] = field(default=False, metadata={"help": "Disable CUDA even when it is available"})
    log_dir: Optional[str] = field(
        default=None,
        metadata={"help": "the directory of the log file. Set to '' to disable logging"}
    )
    seed: Optional[int] = field(
        default=42, metadata={"help": "Random seed that will be set at the beginning of training."}
    )
    debug_mode: Optional[bool] = field(
        default=False, metadata={"help": "Debugging mode with fewer training data"}
    )

    # The following three functions are copied from transformers.training_args
    @cached_property
    @torch_required
    def _setup_devices(self) -> "torch.device":
        if self.no_cuda:
            device = torch.device("cpu")
            self._n_gpu = 0
        else:
            device = torch.device("cuda")
            self._n_gpu = 1

        return device

    @property
    @torch_required
    def device(self) -> "torch.device":
        """
        The device used by this process.
        """
        return self._setup_devices

    @property
    @torch_required
    def n_gpu(self) -> "int":
        """
        The number of GPUs used by this process.

        Note:
            This will only be greater than one when you have multiple GPUs available but are not using distributed
            training. For distributed training, it will always be 1.
        """
        # Make sure `self._n_gpu` is properly setup.
        _ = self._setup_devices
        return self._n_gpu

@dataclass
class CHMMConfig(CHMMArguments):
    """
    Conditional HMM training configuration
    """
    sources: Optional[List[str]] = None
    entity_types: Optional[List[str]] = None
    bio_label_types: Optional[List[str]] = None
    src_priors: Optional[dict] = None
    d_emb: Optional[int] = None

    @cached_property
    def _get_hidden_dim(self) -> "int":
        """
        Returns the HMM hidden dimension, AKA, the number of bio labels
        """
        return len(self.bio_label_types) if self.bio_label_types is not None else 0

    @property
    def d_hidden(self) -> "int":
        return self._get_hidden_dim

    @property
    def d_obs(self) -> "int":
        return self._get_hidden_dim

    @property
    def n_lbs(self):
        """
        Returns
        -------
        The number of BIO labels
        """
        return len(self.bio_label_types) if self.bio_label_types is not None else 0

    @property
    def n_src(self) -> "int":
        """
        Returns
        -------
        The number of sources
        """
        return len(self.sources) if self.sources is not None else 0

    def from_args(self, args: CHMMArguments) -> "CHMMConfig":
        """
        Initialize configuration from arguments

        Parameters
        ----------
        args: arguments (parent class)

        Returns
        -------
        self (type: CHMMConfig)
        """
        logger.info(f'Setting {type(self)} from {type(args)}.')
        arg_elements = {attr: getattr(args, attr) for attr in dir(args) if not callable(getattr(args, attr))
                        and not attr.startswith("__") and not attr.startswith("_")}
        logger.info(f'The following attributes will be changed: {arg_elements.keys()}')
        for attr, value in arg_elements.items():
            try:
                setattr(self, attr, value)
            except AttributeError:
                pass
        return self


import time
from typing import List, Text, NoReturn, Optional

from praw.exceptions import APIException
from praw.models import Submission, Comment, Subreddit
from prawcore import Forbidden
from redlock import RedLockError
from time import perf_counter

from redditrepostsleuth.core.celery.helpers.repost_image import save_image_repost_general
from redditrepostsleuth.core.config import Config
from redditrepostsleuth.core.db.databasemodels import Post, MonitoredSub, MonitoredSubChecks
from redditrepostsleuth.core.db.uow.sqlalchemyunitofworkmanager import SqlAlchemyUnitOfWorkManager
from redditrepostsleuth.core.services.duplicateimageservice import DuplicateImageService
from redditrepostsleuth.core.exception import NoIndexException, RateLimitException, InvalidImageUrlException
from redditrepostsleuth.core.logging import log
from redditrepostsleuth.core.model.events.sub_monitor_event import SubMonitorEvent
from redditrepostsleuth.core.model.search.image_search_results import ImageSearchResults
from redditrepostsleuth.core.services.eventlogging import EventLogging
from redditrepostsleuth.core.services.reddit_manager import RedditManager
from redditrepostsleuth.core.services.response_handler import ResponseHandler
from redditrepostsleuth.core.services.responsebuilder import ResponseBuilder
from redditrepostsleuth.core.util.helpers import build_msg_values_from_search, build_image_msg_values_from_search, \
    save_link_repost, get_image_search_settings_for_monitored_sub, get_link_search_settings_for_monitored_sub
from redditrepostsleuth.core.util.objectmapping import submission_to_post
from redditrepostsleuth.core.util.repost_helpers import get_link_reposts
from redditrepostsleuth.ingestsvc.util import pre_process_post

class SubMonitor:

    def __init__(
            self,
            image_service: DuplicateImageService,
            uowm: SqlAlchemyUnitOfWorkManager,
            reddit: RedditManager,
            response_builder: ResponseBuilder,
            response_handler: ResponseHandler,
            event_logger: EventLogging = None,
            config: Config = None
    ):
        self.image_service = image_service
        self.uowm = uowm
        self.reddit = reddit
        self.response_builder = response_builder
        self.resposne_handler = response_handler
        self.event_logger = event_logger
        if config:
            self.config = config
        else:
            self.config = Config()

    def run(self):
        while True:
            try:
                with self.uowm.start() as uow:
                    monitored_subs = uow.monitored_sub.get_all()
                    for sub in monitored_subs:
                        if not sub.active:
                            log.debug('Sub %s is disabled', sub.name)
                            continue
                        self._check_sub(sub)
                log.info('Sleeping until next run')
                time.sleep(60)
            except Exception as e:
                log.exception('Sub monitor service crashed', exc_info=True)

    def has_post_been_checked(self, post_id: Text) -> bool:
        """
        Check if a given post ID has been checked already
        :param post_id: ID of post to check
        """
        with self.uowm.start() as uow:
            checked = uow.monitored_sub_checked.get_by_id(post_id)
            if checked:
                return True
        return False

    def should_check_post(self, post: Post, check_image: bool, check_link: bool, title_keyword_filter: List[Text] = None) -> bool:
        """
        Check if a given post should be checked
        :rtype: bool
        :param post: Post to check
        :param title_keyword_filter: Optional list of keywords to skip if in title
        :return: bool
        """
        if post.left_comment:
            return False

        if post.post_type not in self.config.supported_post_types:
            return False

        if post.post_type == 'image' and not check_image:
            return False

        if post.post_type == 'link' and not check_link:
            log.info('Skipping link post')
            return False

        if post.crosspost_parent:
            log.debug('Skipping crosspost')
            return False

        if title_keyword_filter:
            for kw in title_keyword_filter:
                if kw in post.title.lower():
                    log.debug('Skipping post with keyword %s in title %s', kw, post.title)
                    return False

        return True

    def check_submission(self, monitored_sub: MonitoredSub, post: Post):
        log.info('Checking %s', post.post_id)
        if post.post_type == 'image' and post.dhash_h is None:
            log.error('Post %s has no dhash', post.post_id)
            return

        try:
            if post.post_type == 'image':
                search_results = self._check_for_repost(post, monitored_sub)
            elif post.post_type == 'link':
                search_results = self._check_for_link_repost(post, monitored_sub)
                if search_results.matches:
                    # TODO - 1/12/2021 - Why are we doing this?
                    save_link_repost(post, search_results.matches[0].post, self.uowm, 'sub_monitor')
            else:
                log.error('Unsuported post type %s', post.post_type)
                return
        except NoIndexException:
            log.error('No search index available.  Cannot check post %s in %s', post.post_id, post.subreddit)
            return
        except RedLockError:
            log.error('New search index is being loaded. Cannot check post %s in %s', post.post_id, post.subreddit)
            return

        if not search_results.matches and monitored_sub.only_comment_on_repost:
            log.debug('No matches for post %s and comment OC is disabled',
                     f'https://redd.it/{search_results.checked_post.post_id}')
            self._create_checked_post(post)
            return

        try:
            comment = self._leave_comment(search_results, monitored_sub)
        except APIException as e:
            error_type = None
            if hasattr(e, 'error_type'):
                error_type = e.error_type
            log.exception('Praw API Exception.  Error Type: %s', error_type, exc_info=True)
            return
        except RateLimitException:
            time.sleep(10)
            return
        except Exception as e:
            log.exception('Failed to leave comment on %s in %s', post.post_id, post.subreddit)
            return

        submission = self.reddit.submission(post.post_id)
        if not submission:
            log.error('Failed to get submission %s for sub %s.  Cannot perform admin functions', post.post_id, post.subreddit)
            return

        if search_results.matches:
            msg_values = build_msg_values_from_search(search_results, self.uowm,
                                                      target_days_old=monitored_sub.target_days_old)
            if search_results.checked_post.post_type == 'image':
                msg_values = build_image_msg_values_from_search(search_results, self.uowm, **msg_values)

            report_msg = self.response_builder.build_report_msg(monitored_sub.name, msg_values)
            self._report_submission(monitored_sub, submission, report_msg)
            self._lock_post(monitored_sub, submission)
            self._remove_post(monitored_sub, submission)
        else:
            self._mark_post_as_oc(monitored_sub, submission)

        self._sticky_reply(monitored_sub, comment)
        self._mark_post_as_comment_left(post)
        self._create_checked_post(post)

    def _check_sub(self, monitored_sub: MonitoredSub):
        log.info('Checking sub %s', monitored_sub.name)
        start_time = perf_counter()
        subreddit = self.reddit.subreddit(monitored_sub.name)
        if not subreddit:
            log.error('Failed to get Subreddit %s', monitored_sub.name)
            return

        submissions = subreddit.new(limit=monitored_sub.search_depth)
        checked_posts = 0
        for submission in submissions:
            if not self.should_check_post(submission):
                continue

            with self.uowm.start() as uow:
                post = uow.posts.get_by_post_id(submission.id)

            if post.post_type == 'image' and post.dhash_h is None:
                log.error('Post %s has no dhash', post.post_id)
                continue
            checked_posts += 1
            try:
                if post.post_type == 'image':
                    search_results = self._check_for_repost(post, monitored_sub)
                elif post.post_type == 'link':
                    search_results = self._check_for_link_repost(post)
            except NoIndexException:
                log.error('No search index available.  Cannot check post %s in %s', submission.id, submission.subreddit.display_name)
                continue
            except RedLockError:
                log.error('New search index is being loaded. Cannot check post %s in %s', submission.id, submission.subreddit.display_name)
                continue

            if not search_results.matches and monitored_sub.only_comment_on_repost:
                log.debug('No matches for post %s and comment OC is disabled',
                         f'https://redd.it/{search_results.checked_post.post_id}')
                self._create_checked_post(post)
                continue

            try:
                comment = self._leave_comment(search_results, submission, monitored_sub)
            except APIException as e:
                error_type = None
                if hasattr(e, 'error_type'):
                    error_type = e.error_type
                log.exception('Praw API Exception.  Error Type: %s', error_type, exc_info=True)
                continue
            except RateLimitException:
                time.sleep(10)
                continue
            except Exception as e:
                log.exception('Failed to leave comment on %s in %s', submission.id, submission.subreddit.display_name)
                continue

            msg_values = build_msg_values_from_search(search_results, self.uowm,
                                                      target_days_old=monitored_sub.target_days_old)
            if search_results.checked_post.post_type == 'image':
                msg_values = build_image_msg_values_from_search(search_results, self.uowm, **msg_values)

            self._sticky_reply(monitored_sub, comment)
            self._lock_post(monitored_sub, submission)
            self._remove_post(monitored_sub, submission)
            self._mark_post_as_oc(monitored_sub, submission)
            self._mark_post_as_comment_left(post)
            self._create_checked_post(post)

            if search_results.matches:
                self._report_submission(monitored_sub, submission)

        process_time = perf_counter() - start_time
        if self.event_logger:
            self.log_run(process_time, checked_posts, monitored_sub.name)

    def _mark_post_as_comment_left(self, post: Post):
        try:
            with self.uowm.start() as uow:
                post.left_comment = True
                uow.posts.update(post)
                uow.commit()
        except Exception as e:
            log.exception('Failed to mark post %s as checked', post.id, exc_info=True)

    def _create_checked_post(self, post: Post):
        try:
            with self.uowm.start() as uow:
                uow.monitored_sub_checked.add(
                    MonitoredSubChecks(post_id=post.post_id, subreddit=post.subreddit)
                )
                uow.commit()
        except Exception as e:
            log.exception('Failed to create checked post for submission %s', post.post_id, exc_info=True)

    def _check_for_link_repost(self, post: Post, monitored_sub: MonitoredSub):
        return get_link_reposts(
            post.url,
            self.uowm,
            get_link_search_settings_for_monitored_sub(monitored_sub),
            post=post,
            get_total=False
        )

    def _check_for_repost(self, post: Post, monitored_sub: MonitoredSub) -> ImageSearchResults:
        """
        Check if provided post is a repost
        :param post: DB Post obj
        :return: None
        """

        search_results = self.image_service.check_image(
            post.url,
            post=post,
            source='sub_monitor',
            search_settings=get_image_search_settings_for_monitored_sub(monitored_sub,
                                                                        target_annoy_distance=self.config.default_annoy_distance)
        )
        if search_results.matches:
            save_image_repost_general(search_results ,self.uowm, 'sub_monitor')

        log.debug(search_results)
        return search_results

    def _sticky_reply(self, monitored_sub: MonitoredSub, comment: Comment):
        if monitored_sub.sticky_comment:
            try:
                comment.mod.distinguish(sticky=True)
                log.info('Made comment %s sticky', comment.id)
            except Forbidden:
                log.error('Failed to sticky comment, no permissions')
            except Exception as e:
                log.exception('Failed to sticky comment', exc_info=True)

    def _remove_post(self, monitored_sub: MonitoredSub, submission: Submission):
        """
        Check if given sub wants posts removed.  Remove is enabled
        @param monitored_sub: Monitored sub
        @param submission: Submission to remove
        """
        if monitored_sub.remove_repost:
            try:
                submission.mod.remove(reason_id=self._get_removal_reason_id(monitored_sub.removal_reason, submission.subreddit))
                log.error('[%s][%s] - Failed to remove post using reason ID %s.  Likely a bad reasons ID', monitored_sub.name, submission.id, monitored_sub.removal_reason_id)
                submission.mod.remove()
            except Forbidden:
                log.error('Failed to remove post https://redd.it/%s, no permission', submission.id)
            except Exception as e:
                log.exception('Failed to remove submission https://redd.it/%s', submission.id, exc_info=True)

    def _get_removal_reason_id(self, removal_reason: Text, subreddit: Subreddit) -> Optional[Text]:
        if not removal_reason:
            return None
        for r in subreddit.mod.removal_reasons:
            if r.title.lower() == removal_reason.lower():
                return r.id
        return None

    def _lock_post(self, monitored_sub: MonitoredSub, submission: Submission) -> NoReturn:
        if monitored_sub.lock_post:
            try:
                submission.mod.lock()
            except Forbidden:
                log.error('Failed to lock post https://redd.it/%s, no permission', submission.id)
            except Exception as e:
                log.exception('Failed to lock submission https://redd.it/%s', submission.id, exc_info=True)

    def _mark_post_as_oc(self, monitored_sub: MonitoredSub, submission: Submission):
        if monitored_sub.mark_as_oc:
            try:
                submission.mod.set_original_content()
            except Forbidden:
                log.error('Failed to set post OC https://redd.it/%s, no permission', submission.id)
            except Exception as e:
                log.exception('Failed to set post OC https://redd.it/%s', submission.id, exc_info=True)

    def _report_submission(self, monitored_sub: MonitoredSub, submission: Submission, report_msg: Text) -> NoReturn:
        if not monitored_sub.report_reposts:
            return
        log.info('Reporting post %s on %s', f'https://redd.it/{submission.id}', monitored_sub.name)
        try:
            submission.report(report_msg)
        except Exception as e:
            log.exception('Failed to report submission', exc_info=True)

    def _leave_comment(self, search_results: ImageSearchResults, monitored_sub: MonitoredSub) -> Comment:

        message = self.response_builder.build_sub_comment(monitored_sub, search_results, )
        return self.resposne_handler.reply_to_submission(search_results.checked_post.post_id, message)

    def save_unknown_post(self, post_id: str) -> Post:
        """
        If we received a request on a post we haven't ingest save it
        :param submission: Reddit Submission
        :return:
        """
        log.info('Post %s does not exist, attempting to ingest', post_id)
        submission = self.reddit.submission(post_id)
        post = None
        try:
            post = pre_process_post(submission_to_post(submission), self.uowm, None)
        except InvalidImageUrlException:
            log.error('Failed to ingest post %s.  URL appears to be bad', post_id)
        if not post:
            log.error('Problem ingesting post.  Either failed to save or it is not an image')
            return

        return post

    def log_run(self, process_time: float, post_count: int, subreddit: str):
        self.event_logger.save_event(
            SubMonitorEvent(
                event_type='subreddit_monitor',
                process_time=process_time,
                post_count=post_count,
                subreddit=subreddit
            )
        )


import numpy as np
from collections import OrderedDict

'''
リカレントニューラルネットワークのモデル
'''
class RecurrentNN:

    def __init__(self,input_size,hidden_size,output_size,weight_init_std = 0.01):
        #重みの初期化
        self.parms = {}
        self.n_input = input_size
        self.n_hidden = hidden_size
        self.n_output = output_size

        #順路
        self.parms['W1'] = weight_init_std * np.random.randn(input_size+1,hidden_size)
        self.parms['WRec'] = weight_init_std * np.random.randn(hidden_size+1,hidden_size)
        self.parms['W2'] = weight_init_std * np.random.randn(hidden_size+1,output_size)

        #レイヤの設定
        self.layers = OrderedDict()
        self.layers['Affine1'] = Affine1(self.parms['W1'])
        self.layers['AffineRec'] = AffineRec(self.parms['W1'],self.parms['WRec'],self.n_hidden)
        self.layers['Sigmoid'] = Sigmoid()
        self.layers['Affine2'] = Affine2(self.parms['W2'],self.parms['WRec'])
        self.lastLayer = TanhWithLoss()

    def forward_all(self,x,z):
        for layer in self.layers.values():
            x,z = layer.forward(x,z)

        return (x,z)

    def forward(self,X):
        z = np.zeros(self.n_hidden)
        zArray,yArray = ([],[])
        for x in X:
            y,z = self.forward_all(x,z)
            zArray.append(z)
            yArray.append(y)

        return (zArray,yArray)

    def gradient(self,x,z,y,t,hidden_delta,output_dEdw,hidden_dEdw,recurr_dEdw):
         dout = 1
         dout = self.lastLayer.backward(dout,x,z,y,t,hidden_delta,output_dEdw,hidden_dEdw,recurr_dEdw)

         layers = list(self.layers.values())
         layers.reverse()
         for layer in layers:
             dout = layer.backward(dout,x,z,y,t,hidden_delta,output_dEdw,hidden_dEdw,recurr_dEdw)

         outs = {}
         outs['dW2'] = output_dEdw.T
         outs['dW1'] = hidden_dEdw.T
         outs['dWRec'] = recurr_dEdw.T

         return outs

    def predict(self, X, Tdata,times):
        zs, ys = self.forward(X)
        zt, yt = self.forward(Tdata)
        y, z = ys[-1], zs[-1]
        for i in range(times):
            y, z = self.forward_all(y, z)
            zs.append(z)
            ys.append(y)

        return (ys,yt)

class Affine1:
    def __init__(self,W1):
        self.W1 = W1

    def forward(self,x,z):
        out = np.dot(np.hstack((1.0, x)),self.W1)

        return (z,out)

    def backward(self,dout,x,z,y,t,hidden_delta,output_dEdw,hidden_dEdw,recurr_dEdw):
        hidden_dEdw += dout.reshape(-1, 1) * np.hstack((1.0, x[t, :]))

class AffineRec:
    def __init__(self,W1,WRec,hidden_size):
        self.W1 = W1
        self.WRec = WRec
        self.n_hidden = hidden_size

    def forward(self,x,z):
        out = z + np.dot(np.hstack((1.0, x)),self.WRec)

        return (x,out)

    def backward(self,dout,x,z,y,t,hidden_delta,output_dEdw,hidden_dEdw,recurr_dEdw):
        zs_prev = z[t - 1] if t > 0 else np.zeros(self.n_hidden)
        recurr_dEdw += dout.reshape(-1, 1) * np.hstack((1.0, zs_prev))

        return dout

class Sigmoid:
    def forward(self, x,z):
        out = sigmoid(z)

        return (z,out)

    def backward(self,dout,x,z,y,t,hidden_delta,output_dEdw,hidden_dEdw,recurr_dEdw):
        dx = dout * (1.0 - z[t]) * z[t]

        return dx

class Affine2:
    def __init__(self,W2,WRec):
        self.W2 = W2
        self.WRec = WRec

    def forward(self,x,z):
        out = tanh(np.dot(np.hstack((1.0, z)),self.W2,))

        return (out,z)

    def backward(self,dout,x,z,y,t,hidden_delta,output_dEdw,hidden_dEdw,recurr_dEdw):
        hidden_delta = (np.dot(dout,self.W2[1:,:].T) + np.dot(hidden_delta,self.WRec[1:,:].T))

        return hidden_delta

class TanhWithLoss:
    def backward(self,dout,x,z,y,t,hidden_delta,output_dEdw,hidden_dEdw,recurr_dEdw):
        output_delta = (y[t] - x[t+1,:]) * (1.0 - y[t] ** 2)
        output_dEdw += output_delta.reshape(-1, 1) * np.hstack((1.0, z[t]))

        return output_delta

#tanh関数
def tanh(z):
    z1 = np.exp(z)
    z2 = np.exp(-z)

    return (z1 - z2) / (z1 + z2)

#シグモイド関数
def sigmoid(x):

    return 1 / (1 + np.exp(-x))


import numpy as np
import matplotlib.pylab as plt

class Activation_function:

	def __init__(self):
		self.x = np.arange(-5.0,5.0,0.1)

	def step_function(self):
		y = np.where(self.x > 0, 1, 0)
		plt.plot(self.x, y)
		plt.ylim(-0.1, 1.1)
		plt.title("Step Function")
		plt.show()

	def sigmoid_function(self):
		y = 1 / (1 + np.exp(-self.x))
		plt.plot(self.x, y)
		plt.ylim(-0.1, 1.1)
		plt.title("Sigmoid Function")
		plt.show()

	def relu_function(self):
		y = np.where(self.x < 0, 0, self.x)
		plt.plot(self.x, y)
		plt.ylim(-0.1, 6)
		plt.title("ReLU Function")
		plt.show()


from PyQt5.QtWidgets import QApplication, QPushButton, QVBoxLayout, QWidget, QMainWindow, QGroupBox, QGridLayout
import datetime

import button_stylesheets
from app import draw

class GUI(QMainWindow):
    def __init__(self):
        super().__init__()
        self.setWindowTitle('GitHub Art')
        self.move(200, 200)
        self.main_widget = QWidget()
        self.main_layout = QVBoxLayout(self.main_widget)
        self.setCentralWidget(self.main_widget)

        self.dates_list = list()
        self.start_day = datetime.datetime(2017, 1, 1, 00, 00)
        day_value = int(self.start_day.strftime('%w'))

        if day_value != 0:
            self.start_day -= datetime.timedelta(days=day_value)

        self.main_layout.addWidget(self.create_grid_layout())

        submit_button = QPushButton('Submit')
        submit_button.clicked.connect(self.get_list_dates)
        self.main_layout.addWidget(submit_button)

        self.show()

    def create_grid_layout(self):
        group_box = QGroupBox("Draw desired graphics here")
        layout = QGridLayout()

        day_count = 0

        for column in range(52):
            for row in range(7):
                date_button = QPushButton(
                    (self.start_day + datetime.timedelta(days=day_count)).strftime('%Y-%m-%d %H:%M:%S')
                )

                date_button.setMinimumSize(15, 15)
                date_button.setMaximumSize(15, 15)
                date_button.setCheckable(True)
                date_button.setStyleSheet(button_stylesheets.UNCHECKED_BUTTON_STYLESHEET)
                date_button.clicked.connect(self.get_button_text)
                layout.addWidget(date_button, row, column)
                day_count += 1

        layout.setSpacing(3)
        group_box.setLayout(layout)
        return group_box

    def get_button_text(self):
        if self.sender().isChecked():
            self.sender().setStyleSheet(button_stylesheets.CHECKED_BUTTON_STYLESHEET)
            self.dates_list.append(self.sender().text())
        else:
            self.sender().setStyleSheet(button_stylesheets.UNCHECKED_BUTTON_STYLESHEET)
            self.dates_list.remove(self.sender().text())

        print(self.sender().text())

    def get_list_dates(self):
        draw(self.dates_list)

if __name__ == '__main__':
    app = QApplication([])
    gui = GUI()
    gui.setFixedSize(gui.size())
    app.exec_()


import os
import numpy as np
import openpyxl
import matplotlib.pyplot as plt
from sklearn import svm
from sklearn.metrics import confusion_matrix
from sklearn.metrics import precision_score, recall_score
from sklearn.externals import joblib
import joblib

#画precision/recall曲线函数
def plot_precision_recall_threshold(precisions,recalls,thresholds):
    plt.plot(thresholds,precisions[:-1],"b--",label="Precision")
    plt.plot(thresholds,recalls[:-1], "g--", label="Recall")
    plt.xlabel("Threshold")
    plt.ylim([0,1])

#画ROC曲线函数
def plot_roc_curve(fpr,tpr,label=None):
    plt.plot(fpr,tpr,linewidth = 2,label = label)
    plt.plot([0,1],[0,1],'k--')
    plt.axis([0,1,0,1])
    plt.xlabel("Flase Positive Rate")
    plt.ylabel("True Positive Rate")

root_path = os.path.dirname(__file__) + os.sep + r"data"
data_path = os.path.dirname(__file__) + os.sep + r"data" + os.sep + r"TrainData"
modal_path = root_path + os.sep + r"model"
if not os.path.exists(modal_path):
    os.makedirs(modal_path)
pr_path = root_path + os.sep + r"prpicture"
if not os.path.exists(pr_path):
    os.makedirs(pr_path)
roc_path = root_path + os.sep + r"rocpicture"
if not os.path.exists(roc_path):
    os.makedirs(roc_path)
TrainPositive = np.load(data_path + os.sep + r"TrainPositive.npy")
TrainNegative = np.load(data_path + os.sep + r"TrainNegative.npy")
TestPositive = np.load(data_path + os.sep + r"TestPositive.npy")
TestNegative = np.load(data_path + os.sep + r"TestNegative.npy")

np.random.seed(42)
PositiveTrainIndex = np.random.choice(np.arange(len(TrainPositive)), 200000)
PositiveTrain = []
for i in PositiveTrainIndex:
    PositiveTrain.append(TrainPositive[i])
PositiveTestIndex = np.random.choice(np.arange(len(TestPositive)), 750)
PositiveTest = []
for i in PositiveTestIndex:
    PositiveTest.append(TestPositive[i])

#Positive加标签
PositiveTrainLabel = []
for i in range(len(PositiveTrain)):
    PositiveTrainLabel.append(1)
PositiveTestLabel = []
for i in range(len(PositiveTest)):
    PositiveTestLabel.append(1)

NegativeTestIndex = np.random.choice(np.arange(len(TestNegative)), 750)
NegativeTest = []
for i in NegativeTestIndex:
    NegativeTest.append(TestNegative[i])
NegativeTestLabel = []
for i in range(len(NegativeTest)):
    NegativeTestLabel.append(-1)

#扩充Negative数量,
NegativeTrain = TrainNegative
temp = TrainNegative
for i in range(9):
    NegativeTrain = np.vstack((NegativeTrain,temp)) # 45060个
NegativeTrainLabel = []
for i in range(len(NegativeTrain)):
    NegativeTrainLabel.append(-1)

#将训练数据合并，并打乱
TrainData_Temp = list(PositiveTrain) + list(NegativeTrain)
TrainLabel_Temp = PositiveTrainLabel + NegativeTrainLabel
TrainIndex = np.arange(len(TrainLabel_Temp ))
TrainIndex = np.random.permutation(TrainIndex)
TrainData = []
TrainLabel = []
for i in TrainIndex:
    TrainData.append(TrainData_Temp[i])
    TrainLabel.append(TrainLabel_Temp[i])

#将测试数据合并，并打乱
TestData_Temp = list(PositiveTest) + list(NegativeTest)
TestLabel_Temp = PositiveTestLabel + NegativeTestLabel
TestIndex = np.arange(len(TestLabel_Temp))
TestIndex = np.random.permutation(TestIndex)
TestData = []
TestLabel = []
for i in TestIndex:
    TestData.append(TestData_Temp[i])
    TestLabel.append(TestLabel_Temp[i])

print(np.array(PositiveTrain).shape)
print(np.array(NegativeTrain).shape)
print(np.array(PositiveTest).shape)
print(np.array(NegativeTest).shape)

print("数据处理完成")

start = 62
# gamma = 0.001,0.01,0.1,1,10,100
gamma = 10
for nu in [0.01, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]:
    print("     ")
    print("     ")
    print("C = {}".format(nu))
    print("     ")
    print("     ")
    #训练
    # clf = SVC(kernel="rbf", C = 1.0, gamma= "auto")
    clf = svm.OneClassSVM(kernel="rbf", gamma=gamma, nu=nu)
    clf.fit(PositiveTrain)
    Modalpath = modal_path + os.sep + r"model" + str(gamma) + "n" + str(nu) + r".plk"
    joblib.dump(clf, Modalpath)
    TestPredict = clf.predict(TestData)
    print("训练完成")
    #评价
    print("返回给定测试集和对应标签的平均准确率")
    n = 0
    for i in range(len(TestPredict)):
        if TestLabel[i] == TestPredict[i]:
            n += 1
    accuracy = n / (len(TestPredict))
    print(accuracy)
    print("混淆矩阵：")
    conf = confusion_matrix(TestLabel,TestPredict)
    print(conf)
    print("precision : ")
    precision = precision_score(TestLabel,TestPredict)
    print(precision)
    print("recall : ")
    recall = recall_score(TestLabel,TestPredict)
    print(recall)
    print("FPR : ")
    FPR = conf[0,1]/(conf[0,1] + conf[0,0])
    print(conf[0,1]/(conf[0,1] + conf[0,0]))

    #交叉评估
    from sklearn.model_selection import cross_val_score
    score = cross_val_score(clf,TestData,TestLabel,cv=3,scoring="accuracy")
    print("cross_val_score ： {}".format(score))

    # precision-recall曲线
    label_score = clf.decision_function(TestData)
    from sklearn.metrics import precision_recall_curve

    precisions, recalls, thresholds = precision_recall_curve(TestLabel, label_score)
    prname = "pr" + str(gamma) + "vs" + str(nu)
    plt.figure("{}".format(prname))
    plot_precision_recall_threshold(precisions, recalls, thresholds)
    prpath = pr_path + os.sep + str(gamma) + "n" + str(nu) + r".png"
    plt.savefig(prpath)
    plt.close()

    # ROC曲线
    from sklearn.metrics import roc_curve

    fpr, tpr, thresholds = roc_curve(TestLabel, label_score)
    rocname = "roc" + str(gamma) + "vs" + str(nu)
    plt.figure("{}".format(rocname))
    plot_roc_curve(fpr, tpr)
    rocpath = roc_path + os.sep + str(gamma) + "n" + str(nu) + r".png"
    plt.savefig(rocpath)
    plt.close()

    #AUC
    from sklearn.metrics import roc_auc_score
    print("AUC : ")
    AUC = roc_auc_score(TestLabel, label_score)
    print(AUC)

    #储存数据到excel
    wb = openpyxl.load_workbook(root_path + os.sep + r"result.xlsx")
    ws = wb.active
    ws["B" + str(start)] = accuracy
    ws["C" + str(start)] = precision
    ws["D" + str(start)] = recall
    ws["E" + str(start)] = FPR
    ws["F" + str(start)] = conf[0,0]
    ws["G" + str(start)] = conf[0,1]
    ws["H" + str(start)] = conf[1,0]
    ws["I" + str(start)] = conf[1,1]
    ws["J" + str(start)] = AUC
    ws["K" + str(start)] = str(score)
    start += 1
    wb.save(root_path + os.sep + r"result.xlsx")


from django.contrib import admin
from mentor.counter.models import Counter
# Register your models here.

class CounterAdmin(admin.ModelAdmin):

	# This changes CounterAdmin list form
	list_display = ('url','timestamp')
	list_filter = ['timestamp']

	date_hierarchy = 'timestamp'
	# This changes CounterAdmin detail form

admin.site.register(Counter, CounterAdmin)


import discord
import os
import random
from keep_alive import keep_alive
import urllib.request, json
import time

from owoify import owoify
import TenGiphPy
# https://github.com/realSnosh/TenGiphPy

from Services import google_search
from Services import quote_search
from Services import to_binary
from datetime import datetime
from Services import eight_ball
from youtube_dl import YoutubeDL
from discord.utils import get
from discord import FFmpegPCMAudio
from Services import reddit_file

# Retrieve keys from .env storage
# (Note, you can replace os.getenv("tokenname") with "key")
discordKey = os.getenv("TOKEN")
# Discord Key is from https://discord.com/developers/applications
# Create bot, then go to Bot -> Token -> Copy
tenorBotKey = os.getenv("TENORKEY")
# Retrieved from https://tenor.com/developer/keyregistration
WolframAppId = os.getenv("APPID")
# Key retrieved from https://products.wolframalpha.com/simple-api/documentation/

tenorBot = TenGiphPy.Tenor(token=tenorBotKey)
client = discord.Client()

################################################
# Variables
################################################

# List of commands
visibleCommands = [
  "inspire (i.e. 'inspire me')", "gif (i.e. 'search a gif for dabbing')", "uwuify (i.e. uwuify to be or not to be)", "youtube" "calculate (i.e. 'calculate the definite integral of x from 0 to 1')",
  "search (i.e. 'search for when discord was made')", "find (i.e. 'find an article about james bond')", "image (i.e. 'find an image of James Bond')", "play (i.e. 'play Astronaut in the Ocean'", "meme"
]

# Indication of words to search for a youtube video
videoIndicatorWords = [
  "youtube",
  "video"
]

# Indication to search for a gif
gifIndicators = [
  "gif",
  "gifs"
]

# Indication of words to get uwu stuff
uwuifyIndicators = [
  "uwuify",
  "owoify"
]

# Indication of meme
memeIndicator =[
  "meme"
]

# Indication words to receive commands
helpindicators = [
  "help",
  "assistance",
  "command",
  "cmds"
]

# Indication words to receive a quote
inspireIndicator = [
  "inspire",
  "quote"
]

# Indication words to calculate or do math
calculateIndicator = [
  "mathematics",
  "math",
  "calculate",
  "calc",
  "solve",
  "wolfram"
]

# Indication words to calculate or do math, but will be kept in the message
calcWhitelistWords = [
  "definite integral",
  "indefinite integral",
  "integrate",
  "derivative",
  "derive",
  "integral",
  "add",
  "subtract",
  "multiply",
  "graph",
  "plot",
  "find max",
  "find min",
  "get max",
  "get min"
]

# Indication words to search for information
searchIndicator =[
  "search"
]

# Indication words to play for music
playIndicator =[
  "play"
]

# Indication words for disconnecting from voice channel
discionnectIndicators = [
  "disconnect",
  "dc",
  "leave",
  "bye"
]

# Indication words to find an article on the internet and share a link
findArticle = [
  "article",
  "link",
  "find"
]

# Indication of image search
findImage = [
  "picture",
  "image",
  "illustration",
  "illustrate",
  "img",
  "pic"
]

# Words that should be removed if they are in front of the indication that is found
slackWords = [
  "in",
  "for",
  "of",
  "to",
  "from",
  "about",
  "search",
  "the",
  "an"
]

def slicer(my_str,sub):
  index=my_str.find(sub)
  if index !=-1 :
    return my_str[(index):]
  else :
    return my_str

def removeIndicators(string, indicatorArray):
  stri = string.lower()
  for val in indicatorArray:
    stri = slicer(stri, val)

  for val in slackWords:
    try:
      index = stri.find(val)
      if index != -1 and index < (val.length()+1):
        stri = stri[val.length():]
    except:
      continue
  print(stri)
  return stri

def help():
	commandString = ""
	commandString += "**Commands** \n"
	commandString += "You can talk to me about the following \n"
	for val in visibleCommands:
		commandString += "- " + val + "\n"
	return commandString
################################################
# Discord Events
################################################

@client.event
async def on_ready():
  print("We have logged in as {0.user}".format(client))
  await client.change_presence(activity=discord.Game("DM me"))

@client.event
async def on_message(message):

	if message.author == client.user:
		return

	msg = message.content

  # If bot is mentioned or in dm's
	mention = f'<@!{client.user.id}>'
	if msg.find(mention) == -1:
		mention = f'<@{client.user.id}>'

	isDM = isinstance(message.channel, discord.DMChannel)
	channel_nsfw = False
	if not isDM:
		channel_nsfw = message.channel.is_nsfw()

	if mention in msg or isDM:
		try:
			if isDM:
				userText = msg
			else:
				userText = msg.split(mention, 1)[1]

			response = ""

			# If another user is mentioned, we would do an 8 ball response
			if len(message.mentions) > 1:
				# print("test")
				try:
					mes = userText.lower()
					if mes.find("what") != -1 or mes.find("where") != -1:
						response = eight_ball.what_ball()
					elif mes.find("who") != -1:
						response = eight_ball.who_ball()
					elif mes.find("why") != -1:
						response = eight_ball.why_ball()
					else:
						response = eight_ball.eight_ball()
				except Exception as err:
					print(err)

			# If Binary, convert to string
			elif to_binary.is_Binary(userText):
				response = to_binary.binaryConvert(userText)

      # If indicated to find an image
			elif any(word in msg.lower() for word in findImage):
				if channel_nsfw:
					try:
						random.seed(time.time())
						response = google_search.image_search(removeIndicators(userText, findImage), False)

					except Exception as err:
						if str(err).find("429") != -1:
							message.channel.send("Img quota exceeded, scraping...")
							response = google_search.image_search2(removeIndicators(userText, findImage), False)
						else:
							response = "Something with image search went wrong."

						print(err)
				else:
					try:
						random.seed(time.time())
						response = google_search.image_search(removeIndicators(userText, findImage), True)
						print(response)

					except Exception as err:
						if str(err).find("429") != -1:
							message.channel.send("Img quota exceeded, scraping...")
							response = google_search.image_search2(removeIndicators(userText, findImage), True)
						else:
							response = "Something with image search went wrong."
						print(err)

      # Disconnect from voice channel
			elif any(word in msg.lower() for word in discionnectIndicators):
				try:
					for x in client.voice_clients:
						if(x.guild == message.guild):
							return await x.disconnect()
					response = "Done :+1:"
				except:
					response = "Something went wrong..."

			# If video/youtube key word, search for a YouTube video or play a youtube song
			elif any(word in msg.lower() for word in videoIndicatorWords) or any(word in msg.lower() for word in playIndicator):
				try:

					toPlayMusic = any(word in msg.lower() for word in playIndicator)
					response = google_search.youtube_search(removeIndicators(removeIndicators(userText, videoIndicatorWords), playIndicator))

					if toPlayMusic:
						await message.channel.send("Loading Song")
						YDL_OPTIONS = {'format': 'bestaudio', 'noplaylist':'True'}
						FFMPEG_OPTIONS = {'before_options': '-reconnect 1 -reconnect_streamed 1 -reconnect_delay_max 5', 'options': '-vn'}
						voice = get(client.voice_clients, guild=message.guild )
						voice_channel = message.author.voice.channel

						if not voice or not voice.is_playing() or voice != voice_channel:
							if voice != voice_channel:
								voice = await voice_channel.connect()

							with YoutubeDL(YDL_OPTIONS) as ydl:
								info = ydl.extract_info(response, download=False)
							URL = info['formats'][0]['url']
							voice.play(FFmpegPCMAudio(URL, **FFMPEG_OPTIONS))
							voice.is_playing()
							await message.channel.send("Playing")
						else:
							await message.channel.send("Already playing song")
							return

				except Exception as err:
					print(err)

      # If indicated to find a meme
			elif any(word in msg.lower() for word in memeIndicator):
				response = reddit_file.get_meme()

      # If indicated as gif
			elif any(word in msg.lower() for word in gifIndicators):
				try:
					random.seed(time.time())
					response = tenorBot.random(removeIndicators(userText, gifIndicators))
				except Exception as err:
					print(err)
      # If indicated to find article
			elif any(word in msg.lower() for word in findArticle):
				try:
					random.seed(time.time())
					links = google_search.get_links(removeIndicators(userText, findArticle))
					response = random.choice(links)
				except Exception as err:
					print(err)

      # If indicated to calculate and do math
			elif any(word in msg.lower() for word in calculateIndicator) or any(word in msg.lower() for word in calcWhitelistWords):
				try:
					textToUse = removeIndicators(userText, calculateIndicator)
					inputVal = textToUse.replace("+", "%2B")
					inputVal = inputVal.replace(" ","+")
					wolfUrl = "https://api.wolframalpha.com/v2/query?input=" + inputVal +"&format=image&output=JSON&appid=" + WolframAppId + "&podstate=Step-by-step%20solution&format=plaintext"

					now = datetime.now()

					current_time = now.strftime("%H:%M:%S")
					print(message.author.name, "requested url", wolfUrl, "at", current_time)
					with urllib.request.urlopen(wolfUrl) as url:
						data = json.loads(url.read().decode())
						if data["queryresult"]:

							await message.channel.send("Note that log is the natural log ln, and there may be absolute values not present when calculating the integral.")
							resp1 = data["queryresult"]["pods"][0]["subpods"][0]["img"]["src"]
							print("Response 1", resp1)
							await message.channel.send(resp1)

							response = data["queryresult"]["pods"][1]["subpods"][0]["img"]["src"]

							resp2 = data["queryresult"]["pods"][0]["subpods"][0]["plaintext"]
							print("Response 2", resp2)
							await message.channel.send(resp2)
							await message.channel.send(data["queryresult"]["pods"][0]["subpods"][1]["img"]["src"])

				except Exception as err:
					print(err)

      # If indicated to search for information
			elif any(word in msg.lower() for word in searchIndicator):
				try:
					textToUse = removeIndicators(userText, calculateIndicator)
					inputVal = textToUse.replace("+", "%2B")
					inputVal = inputVal.replace(" ","+")
					wolfUrl = "https://api.wolframalpha.com/v2/query?input=" + inputVal +"&format=plaintext&output=JSON&appid=" + WolframAppId
					print(wolfUrl)
					with urllib.request.urlopen(wolfUrl) as url:
						data = json.loads(url.read().decode())
						if data["queryresult"]:
							response = data["queryresult"]["pods"][1]["subpods"][0]["plaintext"]

				except Exception as err:
					print(err)

      # If indicated to uwuify text
			elif any(word in msg.lower() for word in uwuifyIndicators):
				try:
					response = owoify(removeIndicators(userText, uwuifyIndicators))
				except Exception as err:
					print(err)

      # If indicated to receive help indications
			elif any(word in msg.lower() for word in helpindicators):
				cmd = help()
				await message.author.send(cmd)

				if not isDM:
					response = message.author.name + ", check your DM's bro"
				else:
					response = "Hope this helps"

      # If indicated to search for a random quote
			elif any(word in msg.lower() for word in inspireIndicator):
				response = quote_search.get_quote()

      # Else, search Google for a response
			else:
				try:
          # First query type tries to find if Google has
          # a system to find resources
					response = google_search.chatbot_query(userText)
					print(response)
				except:
          # If not, try to find our own resource
					response = google_search.chatbot_query2(userText)
			await message.channel.send(response)

		except:
      # Soon gonna add a small-talk AI bot here, if all else doesn't work
			await message.channel.send(
			    "I'm sorry, it appears that something wrong has occured. Please try again."
			)

keep_alive()
client.run(discordKey)