from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import WebDriverException
from selenium.common.exceptions import NoAlertPresentException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as ec
from selenium.webdriver.common.by import By
from CATLogger import SOLog
from EFields import CommonPaths
from Navigation import NavigateSteps
from Utils import ScreenControlDuringExecution
import time

def exists_by_xpath(driver, xpath):
    try:
        driver.find_element_by_xpath(xpath)
    except NoSuchElementException:
        return False
    return True

def is_control_readonly(driver, field_name):
    raw_xpath = CommonPaths.xpath_of_field_readonly_div(field_name)
    readonly_xpath = raw_xpath + '''[contains(@style, "display: block")]'''
    print('readonly_xpath: {x}'.format(x=readonly_xpath))
    readonly_elements = driver.find_elements_by_xpath(readonly_xpath)
    print(len(readonly_elements))
    if len(readonly_elements) > 0:
        return True
    else:
        return False

def is_alert_present(driver):
    try:
        driver.switch_to_alert()
        print('Utils: there is an alert')
        return True
    except NoAlertPresentException:
        print('Utils: there is not an alert')
        return False

def is_help_alert_present(driver):
    try:
        is_element_present(driver, 'id', 'divTreeDialog')
        print('Utils: there is a help alert')
        return True
    except NoAlertPresentException:
        print('Utils: there is not a help alert')
        return False

def accept_help_alert(driver):
    # find OK button in following sibling div
    print('find xpath for button')
    button_xpath = '''//div[@id="divTreeDialog"]/following-sibling::div/button/span[.="OK"]'''
    print(button_xpath)
    button_element = driver.find_element_by_xpath(button_xpath)
    button_element.click()
    print('button-click attempted')
    button = WebDriverWait(driver, 3).until(
        ec.invisibility_of_element_located((By.XPATH, button_xpath))
    )
    print('wait attempted')

def isElementPresent(self, locator):
    try:
        self.driver.find_element_by_xpath(locator)
    except NoSuchElementException:
        print('No such thing')
        return False
    return True

def is_element_present(driver, locator_type, locator):
    driver.implicitly_wait(0)
    if locator_type == 'xpath':
        # print('element checking at xpath: {x}'.format(x=locator))
        try:
            driver.find_element_by_xpath(locator)
            return True
        except NoSuchElementException:
            # print('element not present at that xpath')
            return False
        finally:
            driver.implicitly_wait(2)
            # elements = driver.find_elements_by_xpath(locator)
        # if elements is None:
        #     print('None')
        # # else:
        # #     print('Not None')
        # if len(elements) > 0:
        #     # print('Utils: at least one such element exists')
        #     return True
        # else:
        #     # print('no such element exists')
        #     return False
    elif locator_type == 'id':
        # print('element checking with id: {x}'.format(x=locator))
        try:
            driver.find_element_by_id(locator)
        except NoSuchElementException:
            # print('element not present with that id')
            return False
        return True
        # elements = driver.find_elements_by_id(locator)
        # if len(elements) > 0:
        #     # print('at least one such element exists')
        #     return True
        # else:
        #     # print('no such element exists')
        #     return False
    else:
        print('new uncatered location type...?')

def get_cwp_page_name(driver, print_log_setting=0):
    NavigateSteps.switch_to_main_frame(driver)
    SOLog.pretty_log_data_value('switched to main frame', print_log_setting)
    xpath = '''//div[@id="MainFramePageDiv"]//h1'''
    SOLog.pretty_log_data_value(xpath, print_log_setting)
    main_frame_title = driver.find_element_by_xpath(xpath)
    SOLog.pretty_log_data_value(main_frame_title, print_log_setting)
    name_of_this_page = main_frame_title.get_attribute('innerHTML')
    SOLog.pretty_log_data_value(name_of_this_page, print_log_setting)
    return name_of_this_page

# def py_list_my_checkbox_user_options(csv_string):
def get_user_set_values(csv_string, field_type, log_level):
    # currently just set up for field_type = checkbox, so not tested for yet

    # input string is csv from SQL python coder giving all selected options appearing in the 'seed' form instance
    # example:     """'Bone', 'Brain', 'Skin'"""
    # example:     """'Other', some free text easily checked"""
    set_trimmed_of_extremity_quotes = csv_string[1:]  # takes off leading single quote
    set_with_special_chars_removed = set_trimmed_of_extremity_quotes  # .replace('^', '')
    user_set_to_process = set_with_special_chars_removed.split("', '")  # splits out interior single quotes
    user_set_to_return = []
    user_free_text = ''
    # print('-- x {x}'.format(x=user_set_to_process))
    for user_opt_val_raw in user_set_to_process:
        # print('-- y {x}'.format(x=user_opt_val_raw))
        # detection and treatment of 'other'/free text
        if """Other', """ in user_opt_val_raw:
            user_opt_val = """'Other'"""
            user_free_text = user_opt_val_raw[8:]
            # print('-- a {x}'.format(x=user_opt_val))
            # print('-- b {x}'.format(x=user_free_text))
        elif user_opt_val_raw[-1] == "'":
            user_opt_val = "'" + user_opt_val_raw[:-1] + "'"
            # print('-- c {x}'.format(x=user_opt_val))
        else:
            user_opt_val = "'" + user_opt_val_raw + "'"  # single-quote-wrap ready for use in xpath expressions
            # print('-- d {x}'.format(x=user_opt_val))
        SOLog.pretty_log_field_step(log_level, 3, user_opt_val, 'Utils user_opt_val')
        user_set_to_return.append(user_opt_val)
    SOLog.pretty_log_field_step(log_level, 2, user_set_to_return, 'Utils user_set_to_return')
    SOLog.pretty_log_field_step(log_level, 2, user_free_text, 'Utils user_free_text')
    user_set_to_return.append(user_free_text)
    return user_set_to_return

def py_list_me(csv_string):
    if csv_string[:1] == "'":
        new_string = csv_string[1:]
    else:
        new_string = csv_string
    if new_string[:-1] == "'":
        set_trimmed_of_outer_quotes = csv_string[:-1]  # takes off extremity single quotes
    else:
        set_trimmed_of_outer_quotes = new_string
    set_turned_to_pylist = set_trimmed_of_outer_quotes.split("', '")  # splits out interior single quotes
    return set_turned_to_pylist

def get_page_set_values(driver, field_name, field_type, log_level):
    page_set_values = []
    if field_type == 'checkbox':
        page_set_xpath = '''//input[@label="''' + field_name + '''"]'''
        SOLog.pretty_log_field_step(log_level, 4, page_set_xpath, 'Utils page_set_xpath')
        page_set_elements = driver.find_elements_by_xpath(page_set_xpath)
        for page_set_element in page_set_elements:
            this_id = page_set_element.get_attribute('id')
            this_value = page_set_element.get_attribute('value')
            if 'Other' not in this_id and 'Score' not in this_id:
                page_set_values.append(this_value)
        SOLog.pretty_log_field_step(log_level, 4, page_set_values, 'Utils page_set_values')
    return page_set_values

def get_page_set_elements(driver, field_name, field_type, log_level):
    page_set_elements = []
    if field_type == 'checkbox':
        page_set_xpath = '''//input[@label="''' + field_name + '''"]'''
        SOLog.pretty_log_field_step(log_level, 3, page_set_xpath, 'Utils page_set_xpath')
        page_set_elements_raw = driver.find_elements_by_xpath(page_set_xpath)
        option_attr_set = []
        for page_set_element in page_set_elements_raw:
            option_attr_set = []  # empty it
            this_id = page_set_element.get_attribute('id')
            this_value = page_set_element.get_attribute('value')
            if 'Other' not in this_id and 'Score' not in this_id:
                option_attr_set.append(page_set_element)
                option_attr_set.append(this_id)
                option_attr_set.append(this_value)
                SOLog.pretty_log_field_step(log_level, 4, option_attr_set, 'Utils option_attr_set')
                page_set_elements.append(option_attr_set)
            SOLog.pretty_log_field_step(log_level, 4, page_set_elements, 'Utils page_set_elements')
        SOLog.pretty_log_field_step(log_level, 3, page_set_elements, 'Utils page_set_elements')
    return page_set_elements

def accept_validation_alert(driver):
    try:
        print('boo')
        driver.switch_to.alert()
        print('bah')
        return True
    except:
        print('no alert present')
        return False

def findnth(haystack, needle, n):
    """bobince's solution on SO Dec 2010: find-the-nth-occurrence-of-substring-in-a-string"""
    parts = haystack.split(needle, n+1)
    if len(parts) <= n+1:
        return -1  # there is no nth needle
    return len(haystack)-len(parts[-1])-len(needle)

def ajax_complete(driver):
    """solution from thuannvn on GitHub - check if jQuery.active is 0"""
    try:
        return 0 == driver.execute_script("return jQuery.active")
    except WebDriverException:
        pass

def my_automation_test():
    """solution from thuannvn on GitHub - check if jQuery.active is 0"""
    ff_driver = webdriver.Firefox()

    ff_driver.get("http://domain.tld")

    # wait for ajax items to load
    WebDriverWait(ff_driver, 10).until(
        ajax_complete, "Timeout waiting for page to load")

    assert "ajax loaded string" in ff_driver.page_source

def string_type(string_to_test):
    print('value entered: {x}'.format(x=string_to_test))
    # test for datetime (dd-mmm-yyyy hh:mm format)
    if len(string_to_test) == 17:  # could be a datetime
        # print('len17')
        dd_part = string_to_test[:2]
        frst_sep = string_to_test[2]
        mmm_part = string_to_test[3:6]
        secd_sep = string_to_test[6]
        yyyy_part = string_to_test[7:11]
        spce_part = string_to_test[11]
        hh_part = string_to_test[12:14]
        h_m_sep = string_to_test[14]
        mm_part = string_to_test[15:17]
        # print(dd_part)
        # print(dd_part.isdigit())
        # print(frst_sep)
        # print(mmm_part)
        # print(mmm_part.upper())
        # print(secd_sep)
        # print(yyyy_part)
        # print('')
        # print(hh_part)
        # print(h_m_sep)
        # print(mm_part)
        if dd_part.isdigit() and yyyy_part.isdigit() and hh_part.isdigit() and mm_part.isdigit() and \
                        frst_sep == '-' and secd_sep == '-' and spce_part == ' ' and h_m_sep == ':' and \
                        mmm_part.upper() in (
                                            'JAN', 'FEB', 'MAR',
                                            'APR', 'MAY', 'JUN',
                                            'JUL', 'AUG', 'SEP',
                                            'OCT', 'NOV', 'DEC'):
            return 'datetime'
            # NB - illegitimate dates and times could pass this test (eg, 45-Jan-1066 36:75), so a sense check may be
            # needed on dd hh and mm
        elif string_to_test.isdigit():
            return 'digit'
        elif string_to_test.isalpha():
            return 'alphaword'
        elif string_to_test.replace(' ', '').isalpha():
            return 'alphasentence'
        else:
            return 'mixed'

    elif len(string_to_test) == 11:  # could be a date
        # print('len11')
        dd_part = string_to_test[:2]
        frst_sep = string_to_test[2]
        mmm_part = string_to_test[3:6]
        secd_sep = string_to_test[6]
        yyyy_part = string_to_test[7:11]
        if dd_part.isdigit() and yyyy_part.isdigit() and frst_sep == '-' and secd_sep == '-' and \
            mmm_part.upper() in ('JAN', 'FEB', 'MAR',
                                 'APR', 'MAY', 'JUN',
                                 'JUL', 'AUG', 'SEP',
                                 'OCT', 'NOV', 'DEC'):
            return 'date'
            # NB - illegitimate dates could pass this test (eg, 45-Jan-1066)
        elif string_to_test.isdigit():
            return 'digit'
        elif string_to_test.isalpha():
            return 'alphaword'
        elif string_to_test.replace(' ', '').isalpha():
            return 'alphasentence'
        else:
            return 'mixed'

    elif len(string_to_test) == 8:  # could be a part date - eg, Jun-2017
        # print('len8')
        mmm_part = string_to_test[:3]
        frst_sep = string_to_test[3]
        yyyy_part = string_to_test[4:8]
        if yyyy_part.isdigit() and mmm_part.upper() in ('JAN', 'FEB', 'MAR',
                                                        'APR', 'MAY', 'JUN',
                                                        'JUL', 'AUG', 'SEP',
                                                        'OCT', 'NOV', 'DEC') and frst_sep == '-':
            return 'partdate'
        elif string_to_test.isdigit():
            return 'digit'
        elif string_to_test.isalpha():
            return 'alphaword'
        elif string_to_test.replace(' ', '').isalpha():
            return 'alphasentence'
        else:
            return 'mixed'

    elif len(string_to_test) == 4:  # could be a year-only
        # print('len4')
        if string_to_test.isdigit() and string_to_test[:2] in ('19', '20') and int(string_to_test[-2:] <= 99):
            return 'yearonly'
        elif string_to_test.isdigit():
            return 'digit'
        elif string_to_test.isalpha():
            return 'alphaword'
        elif string_to_test.replace(' ', '').isalpha():
            return 'alphasentence'
        else:
            return 'mixed'

    else:  # len must be other than 17, 11, 8 or 4
        # print('len other')
        if string_to_test.isdigit():
            return 'digit'
        elif string_to_test.isalpha():
            return 'alphaword'
        elif string_to_test.replace(' ', '').isalpha():
            return 'alphasentence'
        else:
            return 'mixed'

def get_page_name(driver):
    h1_xpath = '''//div[@id="MainFramePageDiv"]/h1'''
    mdt_h1_xpath = '''//div[@id="SiteTitle"]/table/tbody/tr[1]/td/h1'''
    delay = 5
    try:
        # main_frame_title = driver.find_element_by_xpath(h1_xpath)
        main_frame_title = WebDriverWait(driver, delay).until(
            ec.presence_of_element_located((By.XPATH, h1_xpath))
        )
    except:
        # main_frame_title = driver.find_element_by_xpath(h1_xpath)
        main_frame_title = WebDriverWait(driver, delay).until(
            ec.presence_of_element_located((By.XPATH, mdt_h1_xpath))
        )
    name_of_this_page = main_frame_title.get_attribute('innerHTML').strip()
    return name_of_this_page

def check_for_jq_dialog(driver, response, delay=5):
    jqdialog_flag = None
    jqdialog_id = 'jqdialog'
    jqdialog_xpath = '''//*[@id="''' + jqdialog_id + '''"]'''
    # print('jqdialog_xpath:', jqdialog_xpath)
    if is_element_present(driver, 'id', jqdialog_id):
        # print('checked if dialog present - yes')
        jqdialog_div = driver.find_element_by_xpath(jqdialog_xpath)
        jqdialog_content = jqdialog_div.get_attribute('innerHTML').replace(
            '* ', '').replace('', '').replace('

', '\n')
        if jqdialog_div.is_displayed():
            print('jqdialog content: {x}'.format(x=jqdialog_content))
            jqdialog_flag = True
        else:
            jqdialog_flag = False
            pass
    else:
        # print('checked if dialog present - no')
        jqdialog_flag = False
        pass

    if jqdialog_flag and response in ('OK', 'Ok', 'ok'):
        jqok_button_xpath = '''//*[@id="jqdialog"]/following-sibling::div/button/span[contains(.,"OK")]'''
        if not is_element_present(driver, 'xpath', jqok_button_xpath):
            print('No visible "{x}" button'.format(x=response))
            time.sleep(30)  # wait while user reviews before proceeding to error
        else:
            # ScreenControlDuringExecution.scroll_to_xpath(driver, jqok_button_xpath)
            print('waiting for {x} seconds, as requested...'.format(x=delay))  # 'delay' is an optionally passed arg
            time.sleep(delay)  # wait while user reviews before moving on by clicking ok
            jqok_button = driver.find_element_by_xpath(jqok_button_xpath)
            jqok_button.click()
            print('clicked "{x}" button'.format(x=response))
            wait = WebDriverWait(driver, 5)  # 'magic' number to wait for the dialog to close
            ui_element = wait.until(ec.invisibility_of_element_located((By.ID, 'jqdialog')))
            print('waited for {x} to disappear'.format(x='jqdialog'))  # print statement can use ui_element (or unused)


"""
Various utility functions
"""
from django.conf import settings
import os, random, hashlib, time, glob, stat

def path_join(*args):
    "Builds absolute path"
    return os.path.abspath(os.path.join(*args))

def uuid(KEY_SIZE=128):
    "Genenerates a unique id"
    id  = str( random.getrandbits( KEY_SIZE ) )
    return hashlib.md5(id).hexdigest()

def cache_file(name=None, ext=None):
    "Generates a path to the cache directory"
    name = name or uuid()
    name = '%s.%s' % (name, ext) if ext else name
    path = path_join(settings.CACHE_DIR, name  )
    return name, path

def cache_clean(name=None, age=1, chance=1):
    "Cleans the cache directory. Chance to trigger, file age in seconds"
    if chance < random.randint(1, 100):
        return
    expiration = time.time()
    fpatt = path_join(settings.CACHE_DIR, '*')
    for fname in glob.glob(fpatt):
        ctime = os.stat(fname)[stat.ST_CTIME ]
        ctime += age * 3600 * 24 * 7 # in days
        if ctime < expiration:
            os.remove(fname)

if __name__ == '__main__':

    cache_clean()


import re
from Graph import Node
from Map import Map
import Drawing as d
import Video as v
from typing import Dict, List, Tuple, Union

r = re.compile(r'\d+ \d+')

def configure_game() -> None:
    """Эта основная функция решаеля. Запрашивает у пользвателя размерность,
    количство пар и координаты конечных точек в каждой паре.
    Ищет решение"""
    size = 0
    pair_count = 0
    pairs = []
    while size < 2 or size > 10:
        try:
            size = int(input('Введите размерность от 2 до 10 \n'))
        except ValueError:
            continue

    while pair_count < 1 or pair_count > size * size // 2:
        try:
            pair_count = int(input('Введите количество пар точек\n'))
        except ValueError:
            continue

    for pair in range(pair_count):
        while True:
            start = input(f'введите координаты первой точки в {pair} паре\n')
            end = input(f'введите координаты второй точки в {pair} паре\n')
            current_pair = is_point_correct(start, end, size)
            if current_pair:
                break
            print('Некорректно, давай по новой')
        pairs.append(current_pair)

    res = find_solution(size, pairs)

    if res[0]:
        color_paths(res[1])
        d.draw_picture(size, res[1])
        v.make_video()

def color_paths(paths: Dict[Tuple[Node, Node], List[Node]]) -> None:
    """Окрашивает все пути в цвет, соответствующий начальной точке"""
    for key in paths:
        color = paths[key][0].color
        for node in paths[key][1:]:
            node.color = color

def find_solution(size: int,
                  pairs: List[Union[Tuple[Tuple[int, int],
                                          Tuple[int, int]], bool]]) \
        -> Tuple[bool, Dict[Tuple[Node, Node], List[Node]] or str]:
    """Эта функция ищет решение головоломки по листу всех пар"""
    i_map = Map(size)
    i_map.make_graph()
    i_map.mark_ends(pairs)
    all_paths = i_map.find_all_paths(pairs)

    if len(all_paths.keys()) < len(pairs):
        return False, 'Нет решений'

    right_paths = find_right_combination(all_paths, i_map.square)
    return right_paths

def find_right_combination(all_path: Dict[Tuple[Node, Node], List[List[Node]]],
                           volume: int):
    """Эта функция ищет корректную комбинацию путей"""
    pairs = list(all_path.keys())
    start_pair = pairs.pop()
    return right_combination_finder(start_pair, set(),
                                    pairs, all_path, volume, {})

def right_combination_finder(current_pair: Tuple[Node, Node], busy_nodes: set,
                             pairs: List[Tuple[Node, Node]],
                             all_paths: Dict[Tuple[Node, Node],
                                             List[List[Node]]],
                             volume: int,
                             right_paths: None or Dict[Tuple[Node, Node],
                                                       List[Node]]) \
        -> Dict[Tuple[Node, Node], List[Node]] or Tuple[bool,
                                                        Dict[Tuple[Node, Node],
                                                             List[Node]]]:
    """Рекурсивный метод комбинирования путей"""
    for path in all_paths[current_pair]:
        set_path = set(path)
        if len(set_path & busy_nodes) != 0:
            continue
        right_paths[current_pair] = path
        busy_nodes |= set_path
        if len(pairs) == 0:
            if len(busy_nodes) == volume:
                return True, right_paths
            else:
                right_paths[current_pair] = None
                busy_nodes -= set_path
                continue
        next_pair = pairs.pop()
        result = right_combination_finder(next_pair,
                                          busy_nodes, pairs, all_paths,
                                          volume, right_paths)
        if not result[0]:
            right_paths[current_pair] = None
            busy_nodes -= set_path
            continue
        else:
            return result

    pairs.append(current_pair)
    return False, right_paths

def is_point_correct(start: str, end: str, size: int) \
        -> Tuple[Tuple[int, int], Tuple[int, int]] or None:
    """Проверка на корректность вводимых точек"""
    if r.match(start) and r.match(end):
        start_x, start_y = start.split()
        end_x, end_y = end.split()
        start_x = int(start_x)
        start_y = int(start_y)
        end_x = int(end_x)
        end_y = int(end_y)
        if not (start_x < 0 or start_x > size
                or start_y < 0 or start_y > size
                or end_x < 0 or end_x > size
                or end_y < 0 or end_y > size):
            return (start_x, start_y), (end_x, end_y)
    return False

if __name__ == '__main__':
    configure_game()


# -*- coding: utf-8 -*-
"""
Created on Tue Oct 23 10:14:36 2018

@author: lzn
"""
import numpy as np
from matplotlib import pyplot as plt
from sklearn import decomposition

# def esd(X):
#     n = X.shape[1] # sample numbers
#     _, D, _ = np.linalg.svd(X / n**0.5)
#     plt.figure()
#     plt.hist(D[:], normed=True) # density distribution
#     plt.show()
#     plt.close()
np.random.seed(1)
X = np.random.randn(500,1000)
m, n = X.shape # sample numbers
_, D, _ = np.linalg.svd(X / n**0.5)

tmppca = decomposition.PCA(svd_solver = 'auto')
# pca = decomposition.PCA()
tmppca.fit(X / n ** 0.5)

d, _  = np.linalg.eigh(np.dot(X, X.T) / n)
if m > n:
    svs = D[m-n:n,]

max_pca = max(D)
min_pca = min(D)
plt.figure(0)
plt.hist(D[:],np.arange((max_pca - min_pca) / 100, max_pca, (max_pca - min_pca) / 100))

plt.figure(1)
# max_pca = max(tmppca.singular_values_)
# min_pca = min(tmppca.singular_values_)
plt.hist(tmppca.singular_values_,np.arange((max_pca - min_pca) / 100, max_pca, (max_pca - min_pca) / 100))
plt.show()
plt.close()


'''
数据加载以及预处理
dataset封装数据集，在使用dataloader实现数据的并行加载
分类:训练集和测试集，从训练集取出一小部分作为验证集
'''
import os
import torch as t
from torch.utils import data
from PIL import Image
import numpy as np
from torchvision import transforms as T

class DogCat(data.Dataset):
    def __init__(self,root,transforms = None,train=True,test=False):
        '''
        获取所有图片的地址，并根据训练，测试，验证三类对数据进行划分
        '''
        self.test = test
        # 拿到所有图片
        imgs = [os.path.join(root,img) for img in os.listdir(root)]
        print(imgs)
        # test1:data/test1/1.jpg
        # train:data/train/cat.1.jpg
        if self.test:
            # 根据不同的分类对图片按序号排序
            imgs = sorted(imgs,key=lambda x:int(x.split('.')[-2].split('/')[-1]))
        else:
            imgs = sorted(imgs,key=lambda x:int(x.split('.')[-2]))
        imgs_nums = len(imgs) # 图片数量，数据集规模

        '''数据集划分'''
        if self.test:
            self.imgs = imgs    # 测试集
        elif train:
            self.imgs = imgs[:int(0.8*(imgs_nums))] # 训练集
        else:
            self.imgs = imgs[int(0.8*imgs_num):] # 验证集

        '''数据增强和转换'''
        if transforms is None:
            normalize = T.Normalize([0.485,0.456,0.406],[0.229,0.224,0.225])

            # 测试集和验证集合:
            if self.test or not train:
                self.transforms = T.Compose([
                    T.Resize(224),
                    T.CenterCrop(224),
                    T.ToTensor(),
                    normalize
                ])
            else:
                # 训练集
                self.transforms = T.Compose([
                    T.Resize(256),
                    T.RandomResizedCrop(224),
                    T.RandomHorizontalFlip(),
                    T.ToTensor(),
                    normalize
                ])

    def __getitem__(self, index):
        '''
        返回一张图片的数据
        :param index:
        :return:如1000.jpg返回1000,如果是训练集和验证集，则对应的是dog返回1，猫则返回0
        '''
        img_path = self.imgs[index]
        if self.test:
            # 是测试集时候
            label = int(self.imgs[index].split('.')[-2].split('/')[-1])
        else:
            label = -1 if 'dog' in img_path.split('/')[-1] else 0
        data = Image.open(img_path)
        data = self.transforms(data)

        return data,label

    def __len__(self):
        '''返回数据集中的所有图片数'''
        return len(self.imgs)

if __name__ == '__main__':
    test = DogCat('/home/gzshan/sgz/dogsVScats/data/test/',test=True)
    print("test:",test)


from rest_framework.test import APITestCase
from rest_framework.views import status
from api.application_form.models import ApplicationForm
from api.application_form.serializers import ApplicationFormSerializer

class ApplicationFormViewSetTest(APITestCase):
    def setUp(self):
        ApplicationForm.objects.create(
            first_name="Bobby",
            last_name="Bass",
            gender="M",
            nationality="USA",
            location="USA",
            email="bbass@rocketmail.com",
            phone="18002254452"
        )

    def test_post_application_form(self):
        self.client.post("/api/v1/applicationForms/", {
            "first_name": "Krikor",
            "last_name": "Ailanjian",
            "gender": "M",
            "nationality": "USA",
            "location": "USA",
            "email": "kailanjian@gmail.com",
            "phone": "6268251906"
        })

        self.assertEqual(2, len(ApplicationForm.objects.all()))

    def test_get_application_form(self):
        response = self.client.get("/api/v1/applicationForms/")

        expected = ApplicationForm.objects.all()
        serialized = ApplicationFormSerializer(expected, many=True)
        self.assertEqual(response.data['results'], serialized.data)
        self.assertEqual(response.status_code, status.HTTP_200_OK)


# https://leetcode.com/explore/challenge/card/january-leetcoding-challenge-2021/581/week-3-january-15th-january-21st/3611/

# Find the Most Competitive Subsequence

# Given an integer array nums and a positive integer k, return the most competitive subsequence of nums of size k.
# An array's subsequence is a resulting sequence obtained by erasing some (possibly zero) elements from the array.
# We define that a subsequence a is more competitive than a subsequence b (of the same length)
# if in the first position where a and b differ, subsequence a has a number less than the corresponding number in b.
# For example, [1,3,4] is more competitive than [1,3,5] because the first position they differ is at the final number,
# and 4 is less than 5.

# Example 1:
# Input: nums = [3,5,2,6], k = 2
# Output: [2,6]
# Explanation: Among the set of every possible subsequence: {[3,5], [3,2], [3,6], [5,2], [5,6], [2,6]}, [2,6] is the most competitive.

# Example 2:
# Input: nums = [2,4,3,3,5,4,9,6], k = 4
# Output: [2,3,3,4]

# Constraints:
# 1 <= nums.length <= 10^5
# 0 <= nums[i] <= 10^9
# 1 <= k <= nums.length

from typing import List
def mostCompetitive(nums: List[int], k: int) -> List[int]:
	n = len(nums)
	stack = []
	lens = 0
	for i, num in enumerate(nums):
		while stack and stack[-1] > num and k - lens < n - i :
			stack.pop()
			lens -= 1
		if lens < k:
			stack.append(num)
			lens += 1
	return stack

assert(mostCompetitive([3,5,2,6], 2) == [2, 6])
assert(mostCompetitive([2,4,3,3,5,4,9,6], 4) == [2, 3, 3, 4])
assert(mostCompetitive([2,4,3,5,5,4,9,6], 5) == [2, 3, 4, 9, 6])

2021 January LeetCoding Challenge /21_mostCompetitive.py


import matplotlib.pyplot as plt
from tikzplotlib import save as tikz_save

filenames = ['basic_vocabulary_data','caused_motion_data','reciprocal_data','topological_relations_data'][::-1]

text = []

for prior in ['dir','LN']:
    for cond in ['False','True']:
        text_ = []
        for fn in filenames:
            f = open('v_measure_{}_{}_{}.tex'.format(fn,prior,cond),'r')
            text_.append(float(f.read().strip()))
        text.append(text_)

marks = ['TR','RE','CM','BV']

plt.plot(text[0],c='#1f77b4')
plt.plot(text[1],c='#1f77b4',alpha=.4)
plt.plot(text[2],c='#ff7f0e')
plt.plot(text[3],c='#ff7f0e',alpha=.4)
plt.xticks([0,1,2,3],marks)

tikz_save('V-measures')


import zipfile
import pickle
import os
import glob

try:
    from xml.etree.cElementTree import XML
except ImportError:
    from xml.etree.ElementTree import XML

# The function required
"""
Module that extract text from MS XML Word document (.docx).
(Inspired by python-docx <https://github.com/mikemaccana/python-docx>)
"""

WORD_NAMESPACE = '{http://schemas.openxmlformats.org/wordprocessingml/2006/main}'
PARA = WORD_NAMESPACE + 'p'
TEXT = WORD_NAMESPACE + 't'

def get_docx_text(path):
    """
    Take the path of a docx file as argument, return the text in unicode.
    """
    document = zipfile.ZipFile(path)
    xml_content = document.read('word/document.xml')
    document.close()
    tree = XML(xml_content)

    paragraphs = []
    for paragraph in tree.getiterator(PARA):
        texts = [node.text
                 for node in paragraph.getiterator(TEXT)
                 if node.text]
        if texts:
            paragraphs.append(''.join(texts))

    return ' '.join(paragraphs)

# The beginning of the loop

with open('nifty_next_tickers.pickle', 'rb') as f:
    tickers = pickle.load(f)

count = 0
for ticker in tickers:
    my_dir = f"C:/Users/SKA/OneDrive/Desktop/annual_reports/"

    my_docx = glob.glob(f"{ticker}/*.docx")

    for file_path in my_docx:

        file_name = file_path.split(".")[0].split("\\")[1]

        if os.path.exists(f"{ticker}\\{file_name}.pickle"):
            print(f"Already have {ticker}\\{file_name}.pickle")

        else:
            try:
                text = get_docx_text(file_path)

                with open(f"{ticker}\\{file_name}.pickle", 'wb') as f:
                    pickle.dump(text, f)
                print(f"Converted {ticker}\\{file_name}.docx to {file_name}.pickle")

            except Exception as e:
                print(f"File at {ticker}\\{file_name}.docx cause a problem")
                print(str(e))
    count += 1
    print(count)


import h5py
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import cm
from plotting_methods import plot_belfiore
import sys

plt.rc('text', usetex=True)
plt.rc('font', family='serif')
plt.rcParams.update({'font.size': 14})

cmap = cm.get_cmap('plasma')
colors_b18 = [cmap(0.85), cmap(0.65), cmap(0.5), cmap(0.3)]
colors_b18 = [cmap(0.8), cmap(0.6), cmap(0.3)]

cmap = cm.get_cmap('viridis')
colors = [cmap(0.85), cmap(0.6), cmap(0.35), cmap(0.15)]
colors = [cmap(0.8), cmap(0.6), cmap(0.3)]

#colors = ['g', 'c', 'b', 'm']
selection = 'gv'
gals = 'cen'
angle = 'rot'
mass_b18 = [10.0,  10.5, 11., 11.5]

data_dir = '/home/sapple/simba_sizes/profiles/paper/plotting/data/'
plot_dir = '/home/sapple/simba_sizes/profiles/paper/plotting/plots/'
bin_labels = ['10.0-10.5', '10.5-11.0', '>11.0']
plot_labels = [r'$10.0 < \textrm{log} (M_* / M_{\odot}) < 10.5$',
                r'$10.5 < \textrm{log} (M_* / M_{\odot}) < 11.0$',
                r'$ \textrm{log} (M_* / M_{\odot}) > 11.0$']
xlabel = r'$ R / R_{half}$'

fig, ax = plt.subplots(2, 2, figsize=(13, 13))
ax = ax.flatten()

# plot SFR:
for i, b in enumerate(bin_labels):
    with h5py.File(data_dir+selection+'_'+angle+'_sfr_data.h5') as f:
        no_gals = f[gals+'_no_gals_'+b].value
        tukey = f[gals+'_tukey_'+b].value
        small = f[gals+'_small_scale_'+b].value
        large = f[gals+'_large_scale_'+b].value
    if i == 0:
        n = len(np.array(tukey))
        dr = 0.2
        factor = dr*n
        bins = np.arange(0., factor, dr)
        rplot = bins+(dr*0.5)

    ax[0].plot(rplot, tukey, color=colors[i], linestyle='-', label=plot_labels[i]+'; '+str(int(no_gals))+' galaxies')
    if i == 1:
        ax[0].fill_between(rplot, tukey - large, tukey + large, color=colors[i], alpha=0.1)
    ax[0].fill_between(rplot, tukey - small, tukey + small, color=colors[i], alpha=0.3)

if gals == 'all':
    ax[0].set_xlim(0, 3.5)
    ax[0].set_ylim(-4.5, -1)
else:
    ax[0].set_xlim(0., 2)
    ax[0].set_ylim(-5.0, -1.5)

ax[0].set_xlabel(xlabel)
ax[0].set_ylabel(r'$ \textrm{log} (\Sigma_{\textrm{SFR}} / M_{\odot}\textrm{yr}^{-1} \textrm{kpc}^{-2})$')

#plot_belfiore(ax[1], selection, colors_b18, mass_b18=mass_b18, label=True)
# plot sSFR:
for i, b in enumerate(bin_labels):
    with h5py.File(data_dir+selection+'_'+angle+'_ssfr_data.h5') as f:
        no_gals = f[gals+'_no_gals_'+b].value
        tukey = f[gals+'_tukey_'+b].value
        small = f[gals+'_small_scale_'+b].value
        large = f[gals+'_large_scale_'+b].value
    if i == 0:
        n = len(np.array(tukey))
        dr = 0.2
        factor = dr*n
        bins = np.arange(0., factor, dr)
        rplot = bins+(dr*0.5)

    ax[1].plot(rplot, tukey, color=colors[i], linestyle='-')
    if i == 1:
        ax[1].fill_between(rplot, tukey - large, tukey + large, color=colors[i], alpha=0.1)
    ax[1].fill_between(rplot, tukey - small, tukey + small, color=colors[i], alpha=0.3)

if gals == 'all':
    ax[1].set_xlim(0, 3.5)
    ax[1].set_ylim(-12.0, -9.5)
else:
    ax[1].set_xlim(0., 2)
    ax[1].set_ylim(-12.5, -10.)
ax[1].set_xlabel(xlabel)
ax[1].set_ylabel(r'$\textrm{log} (\textrm{sSFR} / \textrm{yr}^{-1})$')
#if not gals == 'all':
#    ax[1].legend(fontsize=12, loc=1)
#else:
#    ax[1].legend(fontsize=12, loc=3)

# plot HI:
for i, b in enumerate(bin_labels):
    with h5py.File(data_dir+selection+'_'+angle+'_h1_data.h5') as f:
        no_gals = f[gals+'_no_gals_'+b].value
        tukey = f[gals+'_tukey_'+b].value
        small = f[gals+'_small_scale_'+b].value
        large = f[gals+'_large_scale_'+b].value
    if i == 0:
        n = len(np.array(tukey))
        dr = 0.2
        factor = dr*n
        bins = np.arange(0., factor, dr)
        rplot = bins+(dr*0.5)

    ax[2].plot(rplot, tukey, color=colors[i], linestyle='-', label=plot_labels[i]+'; '+str(int(no_gals))+' galaxies')
    if i == 1:
        ax[2].fill_between(rplot, tukey - large, tukey + large, color=colors[i], alpha=0.1)
    ax[2].fill_between(rplot, tukey - small, tukey + small, color=colors[i], alpha=0.3)

ax[2].set_ylim(4.5,8.)
ax[2].set_xlim(0, 5)
ax[2].set_xlabel(xlabel)
ax[2].set_ylabel(r'$ \textrm{log} (\Sigma_{HI} / M_{\odot}\textrm{kpc}^{-2})$')
if gals == 'all':
    ax[2].legend(fontsize=12, loc=3)
else:
    ax[2].legend(fontsize=12, loc=1)

# plot fmol:
for i, b in enumerate(bin_labels):
    with h5py.File(data_dir+selection+'_'+angle+'_fmol_data.h5') as f:
        no_gals = f[gals+'_no_gals_'+b].value
        tukey = f[gals+'_tukey_'+b].value
        small = f[gals+'_small_scale_'+b].value
        large = f[gals+'_large_scale_'+b].value
    if i == 0:
        n = len(np.array(tukey))
        dr = 0.2
        factor = dr*n
        bins = np.arange(0., factor, dr)
        rplot = bins+(dr*0.5)

    ax[3].plot(rplot, tukey, color=colors[i], linestyle='-')
    if i == 1:
        ax[3].fill_between(rplot, tukey - large, tukey + large, color=colors[i], alpha=0.1)
    ax[3].fill_between(rplot, tukey - small, tukey + small, color=colors[i], alpha=0.3)

ax[3].set_ylim(-1.5, 1.)
ax[3].set_xlim(0, 5)
ax[3].set_xlabel(xlabel)
ax[3].set_ylabel(r'$ \textrm{log} (f_{mol})$')

plt.savefig(plot_dir+selection+'_'+angle+'_all_radial.png')
plt.show()
plt.clf()


from pliers.stimuli.image import ImageStim
from pliers.extractors.base import Extractor, ExtractorResult
import numpy as np
import os
import tempfile
import sys
import requests
import tarfile
from scipy.misc import imsave
import subprocess
import re

class TensorFlowInceptionV3Extractor(Extractor):

    _input_type = ImageStim

    def __init__(self, model_dir=None, data_url=None, num_predictions=5):

        super(TensorFlowInceptionV3Extractor, self).__init__()

        if model_dir is None:
            model_dir = os.path.join(tempfile.gettempdir(), 'TFInceptionV3')
        self.model_dir = model_dir

        if data_url is None:
            data_url = 'http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz'
        self.data_url = data_url

        filename = self.data_url.split('/')[-1]
        self.model_file = os.path.join(self.model_dir, filename)
        self.num_predictions = num_predictions

        # Download the inception-v3 model if needed
        if not os.path.exists(self.model_file):
            self._download_pretrained_model()

    def _download_pretrained_model(self):
        # Adapted from def_maybe_download_and_extract() in TF's classify_image.py
        print("Downloading Inception-V3 model from TensorFlow website...")
        if not os.path.exists(self.model_dir):
            os.makedirs(self.model_dir)
        filename = os.path.basename(self.model_file)
        if not os.path.exists(self.model_file):
            r = requests.get(self.data_url)
            with open(self.model_file, 'wb') as f:
                f.write(r.content)
            size = os.stat(self.model_file).st_size
            print('\tSuccesfully downloaded', filename, size, 'bytes.')
            tarfile.open(self.model_file, 'r:gz').extractall(self.model_dir)

    def _extract(self, stim):
        import tensorflow as tf
        tf_dir = os.path.dirname(tf.__file__)
        script = os.path.join(tf_dir, 'models', 'image', 'imagenet', 'classify_image.py')

        if stim.filename is None:
            img_file = tempfile.mktemp() + '.jpg'
            imsave(img_file, stim.data)
            use_tmp = True
        else:
            img_file = stim.filename
            use_tmp = False

        args = ' --image_file %s --model_dir %s --num_top_prediction %d' % \
                (img_file, self.model_dir, self.num_predictions)
        cmd = ('python ' + script + args).split()
        process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
        output, errors = process.communicate()
        hits = output.decode('utf-8').splitlines()[-self.num_predictions:]

        values, features = [], []
        for i, h in enumerate(hits):
            m = re.search('(.*?)\s\(score\s\=\s([0-9\.]+)\)', h.strip())
            values.extend(m.groups())
            ind = i + 1
            features.extend(['label_%d' % ind, 'score_%d' % ind])

        if use_tmp:
            os.remove(img_file)

        return ExtractorResult([values], stim, self, features=features)


import torch
from torch.autograd import Variable
import numpy as np
import matplotlib.pyplot as plt
import re
import string
import sys

def LocationSign(x, z, radius=1.0, dense=50):
    xx = np.linspace(x - radius, x + radius, dense).astype('float32')
    xx = xx.repeat(dense).reshape(-1, 1)

    zz = np.linspace(z - radius, z + radius, dense).astype('float32')
    zz = np.tile(zz, [dense]).reshape(-1, 1)

    yy = np.zeros(dense * dense).reshape(-1, 1).astype('float32')

    flag = ((np.power(xx - x, 2) + np.power(zz - z, 2)) <= np.power(radius, 2)).reshape(-1)

    xx = xx[flag, :]
    yy = yy[flag, :]
    zz = zz[flag, :]

    coor = np.concatenate((xx, yy, zz), axis=1)

    # color = np.ones((coor.shape[0], 3)).astype('uint8')
    # color[:, 0] = 255

    return coor

def generateTopView(features, depth, size=28, heightThreshold=0.7):
    # features bs * 1025 * 28 * 28, cuda tensor
    # depth bs * 1 * 28 * 28, numpy array

    batchSize, channel, H, W = features.size()
    CAMERA_FACTOR = 4000.0
    topViewSize = size
    imageScale = H / 640.0
    cx = 320.0 * imageScale
    cy = 320.0 * imageScale
    fx = 320.0 * imageScale
    fy = 320.0 * imageScale

    # Cut off depth()
    mask = (depth >= 2.5 * depth.mean())
    depth[mask] = 0.0
    # depth = np.squeeze(depth, 1)
    # Calculate the coordinates
    pz = depth * 65535 / CAMERA_FACTOR

    mesh_px = np.repeat(np.array(range(0, W)).reshape(1, -1), H, axis=0).reshape(1, H, W)
    mesh_px = np.repeat(mesh_px, batchSize, axis=0).astype('float32')
    px = (mesh_px - cx) * pz / fx

    mesh_py = range(0, H)
    mesh_py.reverse()
    mesh_py = np.repeat(np.array(mesh_py).reshape(-1, 1), W, axis=1).reshape(1, H, W)
    mesh_py = np.repeat(mesh_py, batchSize, axis=0).astype('float32')
    py = (mesh_py - cy) * pz / fy

    # pz = -1 * pz

    # py = -1 * py

    px = px.reshape(batchSize, H, W, 1)
    py = py.reshape(batchSize, H, W, 1)
    pz = pz.reshape(batchSize, H, W, 1)

    coor = np.concatenate((px, py, pz), axis=3)
    coor = coor.reshape(batchSize, -1, 3)  # batchSize * 784 * 3

    features = features.contiguous().view(batchSize, channel, -1)  # batch * 1025 *  784
    features = torch.transpose(features, 2, 1)  # batch * 784 * 1025

    # add ego-location sign
    locCoor = LocationSign(0.0, 0.0, radius=0.5, dense=20)
    locCoor = locCoor.reshape(1, -1, 3)
    locCoor_len = locCoor.shape[1]
    locCoor = np.repeat(locCoor, batchSize, axis=0)  # batch * locCoor_len * 3
    coor = np.concatenate((coor, locCoor), axis=1)

    x_min = np.min(coor[:, :, 0], axis=1).reshape(batchSize, 1)
    y_min = np.min(coor[:, :, 1], axis=1).reshape(batchSize, 1)
    z_min = np.min(coor[:, :, 2], axis=1).reshape(batchSize, 1)

    coor[:, :, 0] -= x_min
    coor[:, :, 1] -= y_min
    coor[:, :, 2] -= z_min

    x_max = np.max(coor[:, :, 0], axis=1).reshape(batchSize, 1)
    # y_max = np.max(coor[:, :, 1], axis=1).reshape(batchSize, 1)
    z_max = np.max(coor[:, :, 2], axis=1).reshape(batchSize, 1)

    for i in range(batchSize):
        if x_max[i] > z_max[i]:
            zoom_in = (topViewSize - 2) / x_max[i]
        else:
            zoom_in = (topViewSize - 2) / z_max[i]
        coor[i, :, :] *= zoom_in

    # y_size = np.max(coor[:, :, 1], axis=1).reshape(batchSize, 1)

    coor = np.floor(coor).astype('int32')

    topView = np.zeros((1, topViewSize, topViewSize, channel + 1)).astype(
        'float32')  # channel + 1, since we add a channel indicate ego-location
    topView = np.repeat(topView, batchSize, axis=0)
    topView = Variable(torch.from_numpy(topView))  # batch * 28 * 28 * 1026
    if torch.cuda.is_available():
        topView = topView.cuda()

    # HEIGHT_TRESHOLD = y_size * heightThreshold

    egoLoc_coor = coor[:, -locCoor_len:, :]
    coor = coor[:, :-locCoor_len, :]

    for i in range(batchSize):
        x = coor[i, :, 0].reshape(-1)
        # y = coor[i,:, 1].reshape(-1)
        z = coor[i, :, 2].reshape(-1)

        ego_x = egoLoc_coor[i, :, 0].reshape(-1)
        ego_z = egoLoc_coor[i, :, 2].reshape(-1)

        ii = torch.LongTensor([i])
        x = torch.from_numpy(x).long()
        z = torch.from_numpy(z).long()
        z = topViewSize - 2 - z

        ego_x = torch.from_numpy(ego_x).long()
        ego_z = torch.from_numpy(ego_z).long()
        ego_z = topViewSize - 2 - ego_z

        topView[ii:ii + 1, z, x, :-1] = features[ii:ii + 1, :, :]
        topView[ii:ii + 1, ego_z, ego_x, -1] = 1.0

    topView = topView.transpose(1, 3)
    topView = topView.transpose(2, 3)

    # for i in range(0,128):
    #     plt.imshow(topView[0,i,:,:].data.numpy() * 255)
    #     plt.ioff()
    #     plt.show()

    return topView  # batch * (1024+2) * 28 * 28

def buildObjectCategory(path):
    category = {}
    with open(path, 'r') as fp:
        for line in fp:
            if line[0] == 'i':
                continue

            item = line.split('\t')
            if item[7] == '':
                item[7] = item[2]
            category[int(item[0])] = item[7].replace(' ', '-')
            # category.append([item[0],item[1],item[2],item[7]])

    category[0] = 'unknown'

    return category

def buildRegionCategory(path):
    category = {}
    with open(path, 'r') as fp:
        for line in fp:
            num = ord(line[1])
            name = line[6:-1] if line[-1] == '\n' else line[6:]
            category[num] = name.replace(' ', '-')
    return category

def buildVocaulary(regionCategoryPath, objectCategoryPath):
    regionCategory = buildRegionCategory(regionCategoryPath)
    objectCategory = buildObjectCategory(objectCategoryPath)

    words = [value for (key, value) in regionCategory.items()] + [value for (key, value) in objectCategory.items()]
    words = list(set(words))

    vocabulary = {}
    vocabularyInverse = {}
    for idx, word in enumerate(words):
        vocabulary[word] = idx
        vocabularyInverse[idx] = word

    return vocabulary, vocabularyInverse

def buildEmbeddingVocabulary(path):
    voc = []
    with open(path, 'r') as fp:
        for line in fp:
            if '\n' in line:
                line = line[:-1]
            voc.append(line)

    return voc

def sort_batch(instr_encoding):
    ''' Extract instructions from a list of observations and sort by descending
        sequence length (to enable PyTorch packing). '''
    base_vocab = ['<PAD>', '<UNK>', '<EOS>']
    padding_idx = base_vocab.index('<PAD>')

    seq_tensor = instr_encoding
    seq_lengths = np.argmax(seq_tensor == padding_idx, axis=1)
    seq_lengths[seq_lengths == 0] = seq_tensor.shape[1]  # Full length

    seq_lengths = torch.from_numpy(seq_lengths).long()

    # Sort sequences by lengths
    seq_lengths, perm_idx = seq_lengths.sort(0, True)

    # return Variable(sorted_tensor, requires_grad=False).long().cuda(), \
    #        mask.byte().cuda(), \
    #        list(seq_lengths), list(perm_idx)
    return seq_lengths, perm_idx

class Tokenizer(object):
    ''' Class to tokenize and encode a sentence. '''
    SENTENCE_SPLIT_REGEX = re.compile(r'(\W+)')  # Split on any non-alphanumeric character

    def __init__(self, vocab=None, encoding_length=20):
        self.encoding_length = encoding_length
        self.vocab = vocab
        self.word_to_index = {}
        if vocab:
            for i, word in enumerate(vocab):
                self.word_to_index[word] = i

    def split_sentence(self, sentence):
        ''' Break sentence into a list of words and punctuation '''
        toks = []
        for word in [s.strip().lower() for s in self.SENTENCE_SPLIT_REGEX.split(sentence.strip()) if
                     len(s.strip()) > 0]:
            # Break up any words containing punctuation only, e.g. '!?', unless it is multiple full stops e.g. '..'
            if all(c in string.punctuation for c in word) and not all(c in '.' for c in word):
                toks += list(word)
            else:
                toks.append(word)
        return toks

    def encode_sentence(self, sentence):
        if len(self.word_to_index) == 0:
            sys.exit('Tokenizer has no vocab')
        encoding = []
        for word in self.split_sentence(sentence[::1]):  # not reverse input sentences
            if word in self.word_to_index:
                encoding.append(self.word_to_index[word])
            else:
                encoding.append(self.word_to_index['<UNK>'])
        encoding.append(self.word_to_index['<EOS>'])
        if len(encoding) < self.encoding_length:
            encoding += [self.word_to_index['<PAD>']] * (self.encoding_length - len(encoding))
        return np.array(encoding[:self.encoding_length]).reshape(1,-1)

    def decode_sentence(self, encoding):
        sentence = []
        for ix in encoding:
            if ix == self.word_to_index['<PAD>']:
                break
            else:
                sentence.append(self.vocab[ix])
        return " ".join(sentence[::1])  # not unreverse before output


"""
This is a prototyp of the functionality that a red cross field worker would have
"""
import sys
import time
import asyncio
import json
import pprint
import ast
from invite_new_user import invite_new_user
from tkinter import *
from tkinter import ttk
from tkinter import simpledialog

#MAIN PARAMETERS
from indy import pool, ledger, wallet, did, crypto, anoncreds, pairwise
from indy.error import IndyError

#MAIN PARAMETERS
from settings import *

global pool_handle

async def accept_sovrin_invite():
    try:

        print_log('\n2. Open ledger and get handle\n')
        pool_handle = await get_pool_handle()

        print_log('Start accepting sovrin invite')
        name = 'Alice'
        # new_user Agent
        print_log('Create wallet config ')
        wallet_config_new_user = json.dumps({"id": name +"_wallet"})
        wallet_credentials_new_user = json.dumps({"key": name + "_wallet_key"})

        print_log('Create the new wallet of the affected_person (this will be done on their own device) ')
        try:
            await wallet.create_wallet(wallet_config_new_user, wallet_credentials_new_user)
        except IndyError:
            print('after error')
            await wallet.delete_wallet(wallet_config_new_user, wallet_credentials_new_user)
            await wallet.create_wallet(wallet_config_new_user, wallet_credentials_new_user)

        new_user_wallet = await wallet.open_wallet(wallet_config_new_user, wallet_credentials_new_user)

        # new_user Agent
        (new_user_creator_did, new_user_creator_key) = await did.create_and_store_my_did(new_user_wallet, "{}")
        print_log('new_user creator DID: ', new_user_creator_did)
        print_log('new_user creator key: ', new_user_creator_key)

        print_log("Open the connections request")
        with open('connection_request.json') as f:
            connection_request = json.load(f)

        # new_user Agent
        connection_response = json.dumps({
            'did': new_user_creator_did,
            'verkey': new_user_creator_key,
            'nonce': connection_request['nonce']
        })

        # new_user Agent
        creator_new_user_verkey = await did.key_for_did(pool_handle, new_user_wallet, connection_request['did'])
        print_log('creator new_user verkey: ', creator_new_user_verkey)

        print_log('Anoncryps a connection response and write it to a file (can only be openen by the redcorss issuer)')
        print_log('IRL this should be send through a server or something')
        # new_user Agent
        anoncrypted_connection_response = await crypto.anon_crypt(creator_new_user_verkey, connection_response.encode('utf-8'))
        #print_log('Anoncrypted_connection_response : ', anoncrypted_connection_response)

        f = open('anoncrypted_connection_response', 'wb')
        f.write(anoncrypted_connection_response)
        f.close()

        creator_did_info_json = json.dumps({
            'did': connection_request['did'],
            'verkey': creator_new_user_verkey
        })
        print_log('Store their did')
        #store_their_did_return = await did.store_their_did(new_user_wallet,  creator_did_info_json)
        print_log('Create Pairwise')
        await pairwise.create_pairwise(new_user_wallet, connection_request['did'], new_user_creator_did, 'random_metadata')

        await wallet.close_wallet(new_user_wallet)
        print_log('DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE')

    except IndyError as e:
        print('Error occurred: %s' % e)

async def new_user_creates_did():
    try:
        pool_handle = await get_pool_handle()
        print_log('AP creates a did for the connection with the RC')
        new_user_wallet = await open_ap_wallet('Alice')
        (new_user_did, new_user_key) = await did.create_and_store_my_did(new_user_wallet, "{}")
        print_log('new_user DID: ', new_user_did)
        print_log('new_user key: ', new_user_key)

        with open('connection_request.json') as f:
            connection_request = json.load(f)

        print_log('new_user creates message with DID and verkey')
        # new_user Agent
        new_user_did_info_json = json.dumps({
            'did': new_user_did,
            'verkey': new_user_key
        })

        #print_log("store_their_did_return", store_their_did_return)
        print_log('new_user key: ', new_user_key)

        print_log('new_user authenticates and encrypts the message by calling auth_crypt')

        creator_new_user_verkey = await did.key_for_did(pool_handle, new_user_wallet, connection_request['did'])
        print_log('get pairwise')
        new_user_creator_did =  await pairwise.get_pairwise(new_user_wallet, connection_request['did'])
        print_log(" new_user_creator_did", new_user_creator_did)
        new_user_creator_did = ast.literal_eval(new_user_creator_did)
        new_user_creator_key = await did.key_for_did(pool_handle, new_user_wallet, new_user_creator_did['my_did'])
        # new_user Agent
        print(new_user_wallet, new_user_creator_key, creator_new_user_verkey)
        print_log('auth_crypt')
        authcrypted_new_user_did_info_json = \
            await crypto.auth_crypt(new_user_wallet, new_user_creator_key, creator_new_user_verkey, new_user_did_info_json.encode('utf-8'))

        print_log('auth_crypt a its user information (did and verkey) and write it to a file (can only be openen by the redcorss issuer)')
        print_log('IRL this should be send through a server or something')
        f = open('authcrypted_new_user_did_info_json', 'wb')
        f.write(authcrypted_new_user_did_info_json)
        f.close()

        await wallet.close_wallet(new_user_wallet)

        print_log('DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE')
    except IndyError as e:
        print('Error occurred: %s' % e)

async def respond_to_cred_offer():
    try:
        # Start of alice
        pool_handle = await get_pool_handle()
        new_user_wallet = await open_ap_wallet('Alice')

        print_log('Open Auth crypted cred offer from the rc id issuer')
        file = open("authcrypted_identity_cred_offer_json", "rb")
        authcrypted_identity_cred_offer_json = file.read()
        file.close()

        print_log("I'm assuming we know their did as we're getting message from them, im cheating here by opening this json to get it")
        with open('connection_request.json') as f:
            connection_request = json.load(f)
        rc_ap_did = connection_request['did']

        rc_verkey, identity_cred_offer_json = await auth_decrypt_using_pairwise_storage(pool_handle, new_user_wallet, rc_ap_did, authcrypted_identity_cred_offer_json)

        print_log("identity_cred_offer_json", str(identity_cred_offer_json))

        ap_rc_did = await get_pairwise_string(new_user_wallet, rc_ap_did)
        print_log('\n PA reads schema & cred def from ledger\n')
        # Alice Agent
        get_schema_request = await ledger.build_get_schema_request(ap_rc_did, identity_cred_offer_json['schema_id'])
        print_log("submit_request")
        get_schema_response = await ledger.submit_request(pool_handle, get_schema_request)
        identity_schema = await ledger.parse_get_schema_response(get_schema_response)

        print_log("identity_schema", identity_schema)

        get_cred_def_request = await ledger.build_get_cred_def_request(ap_rc_did, identity_cred_offer_json['cred_def_id'])
        print_log("Na get_cred_def_request")
        get_cred_def_response = await ledger.submit_request(pool_handle, get_cred_def_request)
        print_log("Na get_cred_def_response ")
        (redcross_identity_cred_def_id, redcross_identity_cred_def_json) = await ledger.parse_get_cred_def_response(get_cred_def_response)
        print_log('redcross_identity_cred_def_json', redcross_identity_cred_def_json)

        print_log('\n Alice creates master secret (NO IDEA WHY)\n')
        # Alice Agent

        ap_master_secret_id = await anoncreds.prover_create_master_secret(new_user_wallet, None)
        print_log('ap_master_secret_id: ',ap_master_secret_id)

        print_log('\n Alice creates Identity-credential request\n')

        # Alice Agent
        identity_cred_offer_json = json.dumps(identity_cred_offer_json)
        (identity_cred_request_json, identity_cred_request_metadata_json) = \
            await anoncreds.prover_create_credential_req(new_user_wallet, ap_rc_did, identity_cred_offer_json,
                                                         redcross_identity_cred_def_json, ap_master_secret_id)
        print_log('identity_cred_request_json: ',identity_cred_request_json)
        print_log('identity_cred_request_metadata_json: ',identity_cred_request_metadata_json)

        print_log("Store items on AP device")
        with open('identity_cred_request_metadata_json.json', 'w') as fp:
            json.dump(identity_cred_request_metadata_json, fp)
        with open('redcross_identity_cred_def_json.json', 'w') as fp:
            json.dump(redcross_identity_cred_def_json, fp)

        print_log('\n ###### MAGICALLY SEND identity_cred_request_json FROM ap TO redCross_idIssuer\n')
        print_log('\n In this case we auth encrypt it and store it\n')

        authcrypted_identity_cred_request_json = \
            await auth_crypt_using_pairwise_storage(pool_handle, new_user_wallet, rc_ap_did, rc_verkey, identity_cred_request_json)

        f = open('authcrypted_identity_cred_request_json', 'wb')
        f.write(authcrypted_identity_cred_request_json)
        f.close()

        print_log('\n ###### INTERFACE\n')
        await wallet.close_wallet(new_user_wallet)
        print_log('DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE ')
    except IndyError as e:
        print('Error occurred: %s' % e)

async def AP_store_credential():
    try:
        pool_handle = await get_pool_handle()
        new_user_wallet = await open_ap_wallet('Alice')
        print_log('\n ###### MAGICALLY RECEIVED  identity_cred_json FROM redCross_idIssuer\n')
        file = open("authcrypted_identity_cred_json", "rb")
        authcrypted_identity_cred_json = file.read()
        file.close()

        print_log("I'm assuming we know rc did as we're getting message from them, im cheating here by opening this json to get it")
        with open('connection_request.json') as f:
            connection_request = json.load(f)
        rc_ap_did = connection_request['did']

        print_log('\n Alice decrypts identity credential in wallet\n')

        rc_verkey, identity_cred_json = await auth_decrypt_using_pairwise_storage(pool_handle, new_user_wallet, rc_ap_did, authcrypted_identity_cred_json)

        identity_cred_json = json.dumps(identity_cred_json)

        print_log("Open cred def and req metadata from ap device")
        with open('identity_cred_request_metadata_json.json') as f:
            identity_cred_request_metadata_json = json.load(f)
        with open('redcross_identity_cred_def_json.json') as f:
            redcross_identity_cred_def_json = json.load(f)

        print_log('\n AP stores identity credential in wallet\n')
        # Alice Agent
        await anoncreds.prover_store_credential(new_user_wallet, None, identity_cred_request_metadata_json,
                                                  identity_cred_json, redcross_identity_cred_def_json, None)

        print_log('DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE DONE ')
    except IndyError as e:
        print('Error occurred: %s' % e)

def run_accept_sovrin_invite():
    print_log("The script is starting")
    asyncio.set_event_loop(asyncio.new_event_loop())
    loop = asyncio.get_event_loop()
    loop.run_until_complete(accept_sovrin_invite())
    loop.close()

def run_new_user_creates_did():
    print_log("The script is starting")
    asyncio.set_event_loop(asyncio.new_event_loop())
    loop = asyncio.get_event_loop()
    loop.run_until_complete(new_user_creates_did())
    loop.close()

def run_respond_to_cred_offer():
    print_log("The script is starting")
    asyncio.set_event_loop(asyncio.new_event_loop())
    loop = asyncio.get_event_loop()
    loop.run_until_complete(respond_to_cred_offer())
    loop.close()

def run_AP_store_credential():
    print_log("The script is starting")
    asyncio.set_event_loop(asyncio.new_event_loop())
    loop = asyncio.get_event_loop()
    loop.run_until_complete(AP_store_credential())
    loop.close()

def select_part(input):
    input = int(input)
    if input ==1:
        run_accept_sovrin_invite()
    elif input ==2:
        run_new_user_creates_did()
    elif input ==3:
        run_respond_to_cred_offer()
    elif input ==4:
        run_AP_store_credential()
    else:
        print_log("Please enter a valid input")

print_log("Person Affected interface:")
# Pharsing args one by one
total = len(sys.argv)
if total ==1:
    print_log("Please enter the number of which part you want to run")
    print_log("1. Accept the sovrin invite")
    print_log("2. Create did to communicate with RC")
    print_log("3. Respond to cred offer")
    print_log("4. Store red cross credential")
    input = input("Enter number: ")
    select_part(input)
elif total == 2:
    input = sys.argv[1]
    select_part(input)
else:
    print_log("Please do not enter more than 1 arg")


import datetime
import json
import copy
from Match import Match
from Round import Round
from enum import Enum
import config

class Frame(object):
    def __init__(self, j):
        self.__dict__ = json.loads(j)

class MatchState(Enum):
    WARMUP = 'warmup'
    LIVE = 'live'
    OVER = 'gameover'
    SWAP = 'intermission'

class ParserStat:
    now = datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d_%Hh%M')
    match = Match(now, '')

    def __init__(self):
        self.frame = 0
        self.state = MatchState.WARMUP
        self.round_count = 0
        self.mvps = 0
        self.scores = 0

    def parse_round(self):
        player = self.frame.player
        map = self.frame.map
        side = self.frame.player['team'] if 'team' in self.frame.player else 0
        print('SIDE', side)
        t_score = map['team_t']['score']
        ct_score = map['team_ct']['score']
        nbr = map['round']
        kill = player['state']['round_kills'] if 'round_kills' in player['state'] else 0
        killhs = player['state']['round_killhs'] if 'round_killhs' in player['state'] else 0
        assist = player['state']['round_assists'] if 'round_assists' in player['state'] else 0
        death = player['state']['round_death'] if 'round_death' in player['state'] else 0
        score = player['match_stats']['score'] if 'score' in player['match_stats'] else 0
        mvp = player['match_stats']['mvps'] if 'mvps' in player['match_stats'] else 0
        self.match.add_round(copy.copy(Round(side, t_score, ct_score, nbr, kill, killhs, assist, death, score, mvp)))

    def parse_data(self, post_data):
        self.frame = Frame(post_data)
        if not ('previously' in self.frame.__dict__):
            return 0
        if 'map' in self.frame.__dict__:
            if self.state != MatchState.LIVE and self.frame.round['phase'] == MatchState.LIVE.value:
                print('LIVE')
                self.state = MatchState.LIVE
                self.match.update_map(self.frame.map['name'])
            print('[FRAME]')
            print(self.frame.__dict__)
            if self.frame.map['phase'] == MatchState.WARMUP.value:
                print("warmup")
                return 0
            elif self.state == MatchState.LIVE and self.frame.map['phase'] == MatchState.LIVE.value:
                # First round
                if self.frame.player['steamid'] == config.Steam['yupin_id']:
                    if self.frame.round['phase'] == 'over' and not ('phase' in self.frame.previously['map']):
                        print("[ROUND_OVER]")
                        self.parse_round()
            elif self.state == MatchState.LIVE and self.frame.map['phase'] == MatchState.SWAP.value and not ('phase' in self.frame.previously['map']):
                print('SWAP')
                self.parse_round()
            elif self.state != MatchState.WARMUP and self.frame.map['phase'] == MatchState.OVER.value:
                self.state = MatchState.WARMUP
                self.parse_round()
                # Export vers db
                print("[GAME_OVER]")
                print(self.match.__dict__)
                for rnd in self.match.rounds:
                    print(rnd.__dict__)
                print("END")
                print("END")


#! /bin/python

#Import the necessary methods and libraries
import json
import pandas as pd
import datetime
import pickle
from time import sleep
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream

RESULTS_FOLDER = 'out/data'

#All boxes are of the form W:S:E:N with all values given in
#longitude/latitude. Boxes returned from the google maps API.
CHICAGO_BOX = [ -87.94, 41.64,  -87.52, 42.02]
LA_BOX      = [-118.66, 33.70, -118.16, 34.34]
HOUSTON_BOX = [ -95.79, 29.52,  -95.01, 30.11]

WINDOW_LENGTH_IN_SEC = 600 #Seconds in a storage window

#This is a basic listener that just prints received tweets to stdout.
class TweetListener(StreamListener):
    ts = 0
    tweets = []

    def __init__(self):
        self.ts = datetime.datetime.now()

    # When we receive data, we calculate the difference in timestamp between the
    # current time and the stored value. If it's greater than the window length,
    # we store the current list of tweets and empty it out to start again.
    # Regardless, we then append the new tweet to the list.
    def on_data(self, data):
        t_diff = datetime.datetime.now() - self.ts
        if(t_diff.total_seconds() > WINDOW_LENGTH_IN_SEC):
            ts_string = self.ts.strftime('%d_%m_%y_%H_%M')
            print('Finished period {}'.format(ts_string))
            with open('{}/{}.pickle'.format(RESULTS_FOLDER, ts_string), "wb+") as f:
                pickle.dump(self.tweets, f)
            self.tweets = []
            self.ts = datetime.datetime.now()

        tweet = json.loads(data)
        self.tweets.append(tweet)
        return True

    def on_error(self, status):
        print(status)

if __name__ == '__main__':
#This handles Twitter authetification and the connection to Twitter Streaming API
    listener = TweetListener()
    with open('credentials.pickle', 'rb') as f:
        auth_dict = pickle.load(f)
    auth = OAuthHandler(auth_dict['consumer_key'], auth_dict['consumer_secret'])
    auth.set_access_token(auth_dict['access_token'], auth_dict['access_token_secret'])
    stream = Stream(auth = auth, listener = listener)
    while True:
        try:
            stream.filter(locations=CHICAGO_BOX+LA_BOX+HOUSTON_BOX)
        except:
            sleep(10)


from django.shortcuts import render

# Create your views here.
from django.shortcuts import render

# Create your views here.
from django.http import HttpResponse
from .models import Douban
from django.db.models import Avg

def hello(request):
    return HttpResponse("Welcome to use Django!")

def index(request):
    shorts = Douban.objects.all()
    # 评论数量
    counter = Douban.objects.all().count()

    # 平均星级
    # star_value = T1.objects.values('n_star')
    star_avg = f" {Douban.objects.aggregate(Avg('n_star'))['n_star__avg']:0.1f} "

    # 情感倾向
    sent_avg = f" {Douban.objects.aggregate(Avg('sentiment'))['sentiment__avg']:0.2f} "

    # 正向数量
    queryset = Douban.objects.values('sentiment')
    condtions = {'sentiment__gte': 0.5}
    plus = queryset.filter(**condtions).count()

    # 负向数量
    queryset = Douban.objects.values('sentiment')
    condtions = {'sentiment__lt': 0.5}
    minus = queryset.filter(**condtions).count()
    return render(request, 'result.html', locals())


import re
import pymongo
from datasource.mongo.connection import Connection as mongoConnection
from dazzlingShow.settings import MONGODB, MONGOPORT

class CommonHandler(object):
    """
    可视化图形结果解析器
    """

    def __init__(self, conn, handle_type, data, *args, **kwargs):
        self.conn = conn
        self.handle_type = handle_type
        self.data = data
        self.data_type = kwargs['data_type'] if 'data_type' in kwargs else None

    # def handle_result(self):
    #     handler = self.get_handler()
    #     if not handler:
    #         raise ImportError('不支持的解析类型 %s。' % self.handle_type)
    #     return handler()
    #
    # def get_handler(self):
    #     return getattr(self, 'handler_for_' + str(self.handle_type), False)
    #
    # def handler_for_1(self):
    #
    #     if isinstance(self.data, dict):
    #         result = {}
    #         for field, sql_l in self.data.items():
    #             res = []
    #             for sql_d in sql_l:
    #                 res.append([data.get(list(data.keys())[0]) for data in self.conn.execute(sql_d).values()])
    #             result.setdefault(field, res)
    #     elif isinstance(self.data, list):
    #         result = []
    #         for d in self.data:
    #             index = 0
    #             for key, val in d.items():
    #                 for i, data in enumerate(self.conn.execute(val).values()):
    #                     if index == 0:
    #                         result.append({key: data.get(list(data.keys())[0])})
    #                     else:
    #                         result[i].update({key: data.get(list(data.keys())[0])})
    #                 index += 1
    #     return result

    def handler_for_datatype(self):
        if self.data_type == None:
            result = {'error': 'miss data_type'}

        elif self.data_type == '1' or self.data_type == '5':
            if isinstance(self.conn, mongoConnection):
                result = {}
                result['x'] = []
                result['y'] = []
                if 'x' in self.data and 'y' in self.data:
                    x_dict = self.conn.execute(self.data['x'][0]).values()
                    for x in x_dict:
                        if '_id' in x:
                            x.pop('_id')
                    y_dict = self.conn.execute(self.data['y'][0]).values()
                    for y in y_dict:
                        if '_id' in y:
                            y.pop('_id')
                    result['x'] = x_dict
                    result['y'] = y_dict
                elif 'x' in self.data:
                    x_dict = self.conn.execute(self.data['x'][0]).values()
                    for x in x_dict:
                        if '_id' in x:
                            x.pop('_id')
                    self.data.pop('x')
                    for y in self.data:
                        y_dict = []
                        y_dict = self.conn.execute(self.data[y][0]).values()
                        for y in y_dict:
                            if '_id' in y:
                                y.pop('_id')
                        result['y'].append([list(y.values())[0] for y in y_dict])
                    result['x'] = [list(x.values())[0] for x in x_dict]
                elif 'y' in self.data:
                    y_dict = self.conn.execute(self.data['y'][0]).values()
                    for y in y_dict:
                        if '_id' in y:
                            y.pop("_id")
                    self.data.pop('y')
                    for x in self.data:
                        x_dict = []
                        x_dict = self.conn.execute(self.data[x][0]).values()
                        for x in x_dict:
                            if '_id' in x:
                                x.pop("_id")
                        result['x'].append([list(x.values())[0] for x in x_dict])
                    result['y'] = [list(y.values())[0] for y in y_dict]
            else:
                result = {}
                result['x'] = []
                result['y'] = []
                if 'x' in self.data and 'y' in self.data:
                    x_dict = self.conn.execute(self.data['x'][0]).values()
                    y_dict = self.conn.execute(self.data['y'][0]).values()
                    result['x'] = x_dict
                    result['y'] = y_dict
                elif 'x' in self.data:
                    x_dict = self.conn.execute(self.data['x'][0]).values()
                    self.data.pop('x')
                    for y in self.data:
                        y_dict = []
                        y_dict = self.conn.execute(self.data[y][0]).values()
                        result['y'].append([list(y.values())[0] for y in y_dict])
                    result['x'] = [list(x.values())[0] for x in x_dict]
                elif 'y' in self.data:
                    y_dict = self.conn.execute(self.data['y'][0]).values()
                    self.data.pop('y')
                    for x in self.data:
                        x_dict = []
                        x_dict = self.conn.execute(self.data[x][0]).values()
                        result['x'].append([list(x.values())[0] for x in x_dict])
                    result['y'] = [list(y.values())[0] for y in y_dict]

        elif self.data_type == '2' or self.data_type == '11':
            result = []
            for d in self.data:
                index = 0
                for key, val in d.items():
                    query_dict = self.conn.execute(val).values()
                    for x in query_dict:
                        if '_id' in x:
                            x.pop('_id')
                    for i, data in enumerate(query_dict):
                        if index == 0:
                            result.append({key: data.get(list(data.keys())[0])})
                        else:
                            result[i].update({key: data.get(list(data.keys())[0])})
                    index += 1

        elif self.data_type == '3':
            result = []
            if 'name' in self.data[0]:
                name_dict = self.conn.execute(self.data[0]['name']).values()
                x_dict = self.conn.execute(self.data[0]['x']).values()
                y_dict = self.conn.execute(self.data[0]['y']).values()
                for x in name_dict:
                    if '_id' in x:
                        x.pop('_id')
                for x in x_dict:
                    if '_id' in x:
                        x.pop('_id')
                for x in y_dict:
                    if '_id' in x:
                        x.pop('_id')
                for x in name_dict:
                    result.append({'name': list(x.values())[0], 'value': []})
                    result[name_dict.index(x)]['value'].append(list(x_dict[name_dict.index(x)].values())[0])
                    result[name_dict.index(x)]['value'].append(list(y_dict[name_dict.index(x)].values())[0])
            else:
                for value in self.data[0].values():
                    query_dict = self.conn.execute(value).values()
                    for x in query_dict:
                        if '_id' in x:
                            x.pop('_id')
                    length = len(query_dict)
                    if len(result) < length:
                        for temp in range(length - len(result)):
                            result.append([])
                    for x in range(length):
                        for y in query_dict[x].values():
                            result[x].append(y)

        elif self.data_type == '4':
            # 返回纯文本内容,equary为查询到内容的索引
            content = ''
            textedit = self.data[0].get('textedit') if 'textedit' in self.data[0] else None
            equary = self.data[0].get('equary') if 'equary' in self.data[0] else None
            if not all([textedit, equary]):
                raise BaseException
            query_dict = self.conn.execute(textedit).values()
            if int(equary) - 1 > len(query_dict):
                content = '索引超过查询到的数据'
            else:
                query_dict = query_dict[int(equary) - 1]
                for value in query_dict.values():
                    content += str(value)
            return content

        elif self.data_type == '6' or self.data_type == '9' or self.data_type == '10':
            result = []
            for key, value in self.data[0].items():
                query_dict = self.conn.execute(value).values()
                length = len(query_dict)
                if len(result) < length:
                    for temp in range(length - len(result)):
                        result.append({})
                for x in query_dict:
                    for y in x.values():
                        result[query_dict.index(x)][key] = y

        elif self.data_type == '7':
            result = []
            textedit = self.data[0].get('textedit') if 'textedit' in self.data[0] else None
            if textedit is None:
                raise BaseException
            query_dict = self.conn.execute(textedit).values()
            for x in query_dict:
                if '_id' in x:
                    x.pop('_id')
                for temp in x.values():
                    result.append(str(temp))

        elif self.data_type == '8':
            result = {}
            for key, value in self.data.items():
                query_dict = self.conn.execute(value).values()[0]
                if '_id' in query_dict:
                    query_dict.pop('_id')
                for x in query_dict.values():
                    result[key] = x

        elif self.data_type == '13':
            mongo = pymongo.MongoClient(MONGODB, MONGOPORT)
            db = mongo.dazzlingshow
            result = []
            for temp in self.data:
                query_str = self.conn.execute(temp['name']).values()
                for x in query_str:
                    if '_id' in x:
                        query_str_1 = self.conn.execute(temp['value']).values()[0]
                        query_str_1.pop('_id')
                        redata = db.area.find_one({'name': x['region_name']}, {'longitude': 1, 'latitude': 1, '_id': 0})
                        if redata:
                            result.append({'name': x['region_name'], 'value': [redata['longitude'], redata['latitude'],
                                                                               list(query_str_1.values())[0]]})
                    else:
                        redata = db.area.find_one({'name': x['region_name']}, {'longitude': 1, 'latitude': 1, '_id': 0})
                        if redata:
                            result.append({'name': x['region_name'], 'value': [redata['longitude'], redata['latitude'],
                                                                               list(self.conn.execute(
                                                                                temp['value']).values()[0].values())[
                                                                                0]]})

        elif self.data_type == '12':
            if 'year1' not in self.data[0]:
                result = []
                count = []

                # 从前端接收到数据重新组织数据结构
                for x in self.data[0]:
                    val = re.search(r'^name(\d{1})$', x)
                    if val:
                        count.append(int(val.group(1)))
                    else:
                        continue
                max_count = max(count)
                data = []
                for x in range(max_count):
                    data.append(
                        {'name': self.data[0]['name' + str(x + 1)], 'value': self.data[0]['value' + str(x + 1)]})
                self.data = data
                for x in self.data:
                    result.append([])
                index = 0
                for x in self.data:
                    index_1 = 0
                    name_list = self.conn.execute(x['name']).values()
                    value_list = self.conn.execute(x['value']).values()
                    for y in name_list:
                        if "_id" in y:
                            y.pop('_id')
                    for y in value_list:
                        if "_id" in y:
                            y.pop('_id')
                    for y in name_list:
                        if index_1 >= 2 or index_1 >= len(value_list):
                            break
                        else:
                            result[index].append(
                                {"name": list(y.values())[0],
                                 'value': list(value_list[name_list.index(y)].values())[0]})
                        index_1 += 1
                    index += 1
            else:
                result = []
                count = []
                # 从前端接收数据重新组织数据结构
                for x in self.data[0]:
                    val = re.search(r'^name(\d{1})$', x)
                    if val:
                        count.append(int(val.group(1)))
                    else:
                        continue
                max_count = max(count)
                data = []
                for x in range(max_count):
                    data.append(
                        {'name': self.data[0]['name' + str(x + 1)], 'value': self.data[0]['value' + str(x + 1)],
                         'year': self.data[0]['year' + str(x + 1)]})
                self.data = data
                for x in self.data:
                    result.append([])
                index = 0
                for x in self.data:
                    index_1 = 0
                    name_list = self.conn.execute(x['name']).values()
                    value_list = self.conn.execute(x['value']).values()
                    for y in name_list:
                        if "_id" in y:
                            y.pop('_id')
                    for y in value_list:
                        if "_id" in y:
                            y.pop('_id')
                    for y in name_list:
                        if index_1 >= 4 or index_1 >= len(value_list):
                            break
                        else:
                            result[index].append(
                                {"name": list(y.values())[0], 'value': list(value_list[name_list.index(y)].values())[0],
                                 "year": x['year']})
                        index_1 += 1
                    index += 1
        else:
            result = {'error': 'unknow error'}

        return result


import os
from search_tags_count import SearchTagsCount
from elasticsearch import Elasticsearch, RequestsHttpConnection
from requests_aws4auth import AWS4Auth

awsauth = AWS4Auth(
    os.environ['AWS_ACCESS_KEY_ID'],
    os.environ['AWS_SECRET_ACCESS_KEY'],
    os.environ['AWS_REGION'],
    'es',
    session_token=os.environ['AWS_SESSION_TOKEN']
)
elasticsearch = Elasticsearch(
    hosts=[{'host': os.environ['ELASTIC_SEARCH_ENDPOINT'], 'port': 443}],
    http_auth=awsauth,
    use_ssl=True,
    verify_certs=True,
    connection_class=RequestsHttpConnection
)

def lambda_handler(event, context):
    search_tags_count = SearchTagsCount(event, context, elasticsearch=elasticsearch)
    return search_tags_count.main()


import os

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, joinedload
from sqlalchemy.orm.exc import NoResultFound

from database_setup import Base, Category, Item, User

class ItemDao(object):

    def __init__(self):
        """Initialize ItemDao.

        This object will be initialized to read/write Items to an sqlite database
        saved to item_catalog.db.
        """

        self.engine = create_engine("sqlite:///item_catalog.db")
        Base.metadata.bind = self.engine
        self.DBSession = sessionmaker(bind=self.engine)

    def close(self):

        self.engine.dispose()

    def createUser(self, login_session):
        newUser = User(name=login_session['username'], email=login_session['email'], picture=login_session['picture'])
        session = self.DBSession()
        session.add(newUser)
        session.commit()
        user = session.query(User).filter_by(email=login_session['email']).one()
        return user.id

    def getUserInfo(self, user_id):
        session = self.DBSession()
        user = session.query(User).filter_by(id=user_id).one()
        return user

    def getUserID(self, email):
        try:
            session = self.DBSession()
            user = session.query(User).filter_by(email=email).one()
            return user.id
        except:
            return None

    def create_item(self, item):
        """Persist a new item to the database."""

        session = self.DBSession()
        session.add(item)
        session.commit()
        session.refresh(item)
        session.close()

        return item

    def edit_item(self, item_to_edit):
        """Edit the properties of an existing item in the database."""

        session = self.DBSession()

        item = self.get_item(item_to_edit.id, session = session)
        item.name = item_to_edit.name
        item.category_id = item_to_edit.category_id
        item.description = item_to_edit.description

        session.add(item)
        session.commit()
        session.refresh(item)
        session.close()

        return item

    def delete_item(self, item):
        """Delete an item from the database."""

        session = self.DBSession()

        item_to_delete = self.get_item(item.id, session = session)

        session.delete(item_to_delete)
        session.commit()
        session.close()

        return item

    def get_items(self, category_id = None):
        """Get all items in the database."""

        session = self.DBSession()
        items = None

        if category_id is None:
            items = session.query(Item).all()
        else:
            items = session.query(Item).filter_by(category_id = category_id).all()

        session.close()

        return items

    def get_item(self, item_id, session = None):
        """Get a specific item in the database by id."""

        is_session_open = False

        if session is not None:
            is_session_open = True

        if is_session_open == False:
            session = self.DBSession()

        item = None
        try:
            item = session.query(Item).filter_by(id = item_id).one()
        except NoResultFound:
            item = None

        if is_session_open == False:
            session.close()
        return item

    def create_category(self, category):
        """Create a new product category."""

        session = self.DBSession()
        session.add(category)
        session.commit()
        session.refresh(category)
        session.close()

        return category

    def get_categories(self, include_items = False):
        """Get all product categories."""

        session = self.DBSession()
        categories = session.query(Category).all()

        # lazy load items
        if include_items == True:
            for category in categories:
                category.items

        session.close()
        return categories

    def get_category(self, category_id, session = None):
        """Get a specific category."""

        is_session_open = False
        if session is not None:
            is_session_open = True

        if is_session_open == False:
            session = self.DBSession()

        category = None
        try:
            category = session.query(Category).filter_by(id = category_id).one()
        except NoResultFound:
            category = None

        if is_session_open == False:
            session.close()

        return category

    def edit_category(self, category_to_edit):
        """Edit a category."""

        session = self.DBSession()

        category = self.get_category(category_to_edit.id, session = session)

        category.name = category_to_edit.name
        session.add(category)
        session.commit()
        session.refresh(category)
        session.close()

        return category

    def delete_category(self, category_to_delete):
        """Delete a category."""

        session = self.DBSession()

        category = self.get_category(category_to_delete.id, session = session)
        session.delete(category)
        session.commit()
        session.close()

        return category

def test_create_item(item_dao):

    item = Item()
    item.name = "Nintendo Switch"

    item_dao.create_item(item)

def test_get_items(item_dao):

    items = item_dao.get_items()

    assert len(items) == 1
    assert items[0].name == "Nintendo Switch"

def test_get_item(item_dao):

    item = item_dao.get_item(1)

    assert item.name == "Nintendo Switch"

def test_get_item_with_no_result_found(item_dao):

    item = item_dao.get_item(100)

    assert item is None

def test_edit_item(item_dao):

    item = Item()
    item.id = 1
    item.name = "SNES Classic"

    item_dao.edit_item(item)

    assert item_dao.get_item(1).name == "SNES Classic"

def test_delete_item(item_dao):

    item = Item()
    item.id = 1

    item_dao.delete_item(item)

    assert item_dao.get_item(1) is None

def test_create_category(item_dao):

    category = Category()

    category.id = 1
    category.name = "Video Games"

    item_dao.create_category(category)

def test_get_categories(item_dao):

    categories = item_dao.get_categories()

    assert len(categories) == 1
    assert categories[0].name == "Video Games"

def test_get_category(item_dao):

    category = item_dao.get_category(1)

    assert category is not None
    assert category.name == "Video Games"

def test_edit_category(item_dao):

    category = Category()

    category.id = 1
    category.name = "Video Game Consoles"

    item_dao.edit_category(category)

    category = item_dao.get_category(1)

    assert category is not None
    assert category.name == "Video Game Consoles"

def test_delete_category(item_dao):

    category = Category()
    category.id = 1

    item_dao.delete_category(category)

    category = item_dao.get_category(1)

    assert category is None

if __name__ == "__main__":

    item_dao = ItemDao()

    test_create_item(item_dao)
    test_get_items(item_dao)
    test_get_item(item_dao)
    test_get_item_with_no_result_found(item_dao)
    test_edit_item(item_dao)
    test_delete_item(item_dao)

    test_create_category(item_dao)
    test_get_categories(item_dao)
    test_get_category(item_dao)
    test_edit_category(item_dao)
    test_delete_category(item_dao)

    item_dao.close()
    os.remove("item_catalog.db")


import httplib, datetime, time
from app import models, db
from thread import start_new_thread
from flask_socketio import emit
import logging

logging.basicConfig()

class Controller():

    # Routine for polling status codes for resources
    def update_all_statuses(self):
        resources = models.Resource.query.all()
        changes_occurred = False
        for r in resources:
            resp = self.get_status(r.url)
            # Save only changes to status table
            if int(resp) != int(r.httpstatus):
                changes_occurred = True
                r.httpstatus = resp
                dt = datetime.datetime.now()
                status = models.Status(r.httpstatus, dt, r.id )
                db.session.add(r)
                db.session.add(status)
                db.session.commit()
        return changes_occurred

    def get_status(self, url):
        try:
            conn = httplib.HTTPConnection(url)
            conn.request('GET','/')
            return conn.getresponse().status
        except:
            return 404


from keras.layers import *
from keras.models import *
from model.model_basic import BasicDeepModel
from keras.utils.vis_utils import plot_model
from keras import regularizers
from keras.layers.core import Masking

min_win = 2
max_win = 2

class SimpleCNNModel(BasicDeepModel):
    def __init__(self, name='basicModel', num_flods=5, model_type=1, config=None):
        self.model_type = model_type
        name = 'simpleCNN' + str(model_type) + config.main_feature
        BasicDeepModel.__init__(self, name=name, n_folds=num_flods, config=config)

    def create_model(self):
        dropout_p = 0.25
        trainable = True

        char_input = Input(shape=(self.char_max_len,), name='char')
        word_input = Input(shape=(self.word_max_len,), name='word')

        c_mask = Masking(mask_value=self.char_mask_value)(char_input)
        w_mask = Masking(mask_value=self.word_mask_value)(word_input)
        char_embedding = Embedding(self.max_c_features, self.char_embed_size, weights=[self.char_embedding], trainable=True, name='char_embedding')
        word_embedding = Embedding(self.max_w_features, self.word_embed_size, weights=[self.word_embedding], trainable=True, name='word_embedding')

        if self.config.main_feature == 'char':
            embedding = char_embedding
            input = char_input
            x = c_mask
        else:
            embedding = word_embedding
            input = word_input
            x = w_mask

        filter_nr = 100

        x = embedding(x)
        #  x = BatchNormalization()(x)
        #  x = SpatialDropout1D(dropout_p)(x)
        kernels = []
        for i in range(min_win, max_win+1):
            cnn = Conv1D(filter_nr, i, padding="valid", strides=1, activation='relu')(x)
            cnn = MaxPooling1D(pool_size=self.char_max_len - i + 1)(cnn)
            cnn = Flatten()(cnn)
            kernels.append(cnn)

        if self.config.main_feature == 'all':
            recurrent_units = 60
            word_embedding_layer = word_embedding(word_input)
            word_embedding_layer = BatchNormalization()(word_embedding_layer)
            word_embedding_layer = SpatialDropout1D(0.2)(word_embedding_layer)
            word_rnn_1 = Bidirectional(CuDNNGRU(recurrent_units // 2, return_sequences=True))(word_embedding_layer)
            word_rnn_1 = SpatialDropout1D(0.2)(word_rnn_1)
            word_rnn_2 = Bidirectional(CuDNNGRU(recurrent_units // 2, return_sequences=True))(word_rnn_1)
            word_maxpool = GlobalMaxPooling1D()(word_rnn_2)
            word_average = GlobalAveragePooling1D()(word_rnn_2)
            concat2 = concatenate([cnn1, cnn2, cnn3, cnn4, word_maxpool, word_average], axis=-1)
            dense2 = Dense(self.n_class, activation="softmax")(concat2)
            res_model = Model(inputs=[char_input, word_input], outputs=dense2)

        else:
            # x = concatenate(kernels, axis=1)
            x = kernels[0]
            #  x = Dropout(dropout_p)(all_views)
            # dense2 = Dense(self.n_class, activation="softmax", kernel_regularizer=regularizers.l2(self.wd))(x)
            dense2 = Dense(self.n_class, activation="softmax")(x)
            res_model = Model(inputs=[input], outputs=dense2)

        plot_model(res_model, to_file="{}.png".format(self.name), show_shapes=True)
        return res_model


from bs4 import BeautifulSoup as bs
import requests
import  csv

def write_csv(data):
    with open('plugins.csv', 'a', encoding='utf-8', newline='\n') as f:
        writer = csv.writer(f)
        writer.writerow([data['url'],
                         data['name']])

def get_pagy(html):
    bs_data = bs(html, 'lxml')
    news = bs_data.find('div', class_='pagination-wrap').find('ul').find_all('li')
    for new in news:
        data = new.find('a').text
        if str(data) == '»':
            datas = new.find('a').get('href')
            datas = str(datas).split('=')
            return int(datas[-1])

def get_data(html):
    bs_data = bs(html, 'lxml')
    # print(dir(bs_data))
    news = bs_data.find('div', class_='container').find('section', class_='row').find('div', class_='news-page').find_all('div', class_='row news_pack')
    # print(news[0].find_all('div', class_='col-xs-12 col-sm-6')[1])
    for new in news:
        datas = new.find_all('div', class_='col-xs-12 col-sm-6')[1].find('div', class_='news_text_col').find('ul', class_='links-list').find_all('li')
        # print(datas)
        for da in datas:
            url = da.find('div').find('a').get('href')
            name = da.find('div').find('a').get('title')
            url = str(url).strip()
            name = str(name).strip()

            data = {
                'url': url,
                'name': name
            }
            write_csv(data)

def get_html(url):
    html = requests.get(url).text
    return html

def main():
    # get_data(get_html('https://www.retail.ru/news/?year=2018&PAGEN_1=1'))

    for y in range(2008, 2019):
        a = get_pagy(get_html(f'https://www.retail.ru/news/?year={y}&PAGEN_1=1'))
        for x in range(a):
            x += 1
            get_data(get_html(f'https://www.retail.ru/news/?year={y}&PAGEN_1={x}'))

if __name__ == '__main__':
    main()