import sys

from elixir.statements import STATEMENTS
from elixir import Entity
from elixir import has_field, belongs_to, has_many
from elixir import Integer, String, CHAR, Integer

from sqlalchemy import func, select

from cartography import geometry

from byCycle.util import gis, joinAttrs
from byCycle.model.entities.util import cascade_args
from byCycle.model.data.sqltypes import POINT, LINESTRING

__all__ = ['Node', 'Edge', 'StreetName', 'City', 'State', 'Place']

def base_statements(base_entity_name):
    """Pseudo-statement to import the statements from a "base" entity.

    When this pseudo-statement is added as the first statement in an
    ``Entity`` class definition, it effectively copies the statements from the
    "base" ``Entity`` identified by ``base_entity_name`` to the "child"
    ``Entity`` that executes the statement.

    """
    entity = globals()[base_entity_name]
    class_locals = sys._getframe(1).f_locals
    statements = class_locals[STATEMENTS] = getattr(entity, STATEMENTS)[:]

class Node(Entity):
    has_field('geom', POINT(4326))
    has_many('edges_f', of_kind='Edge', inverse='node_f')
    has_many('edges_t', of_kind='Edge', inverse='node_t')

    @property
    def edges(self):
        return list(self.edges_f) + list(self.edges_t)

class Edge(Entity):
    has_field('addr_f_l', Integer)
    has_field('addr_f_r', Integer)
    has_field('addr_t_l', Integer)
    has_field('addr_t_r', Integer)
    has_field('even_side', CHAR(1)),
    has_field('one_way', Integer)
    has_field('permanent_id', Integer)
    has_field('geom', LINESTRING(4326))
    belongs_to('node_f', of_kind='Node', **cascade_args)
    belongs_to('node_t', of_kind='Node', **cascade_args)
    belongs_to('street_name', of_kind='StreetName', **cascade_args)
    belongs_to('place_l', of_kind='Place', **cascade_args)
    belongs_to('place_r', of_kind='Place', **cascade_args)

    def to_feet(self):
        return self.to_miles() * 5280.0

    def to_miles(self):
        return gis.getLengthOfLineString([self.geom.pointN(n) for n in
                                          range(self.geom.numPoints())])

    def to_kilometers(self):
        return self.to_miles() * 1.609344

    def to_meters(self):
        return self.to_kilometers() / 1000.0

    def getSideNumberIsOn(self, num):
        """Determine which side of the edge, "l" or "r", ``num`` is on."""
        # Determine odd side of edge, l or r, for convenience
        odd_side = ('l', 'r')[self.even_side == 'l']
        # Is ``num`` on the even or odd side of this edge?
        # FIXME: What if there's no address range on the side ``num`` is on?
        #        Right now, we return the odd side by default
        return (odd_side, self.even_side)[int(num) % 2 == 0]

    def getPlaceOnSideNumberIsOn(self, num):
        """Get `Place` on side ``num`` is on."""
        side = self.getSideNumberIsOn(num)
        if side == 'l':
            return self.place_l
        else:
            return self.place_r

    def __len__(self):
        """Get the length of this `Edge`, using cached value if available."""
        try:
            self._length
        except AttributeError:
            self._length = self.geom.length()
        return self._length

    length = __len__

    def getPointAndLocationOfNumber(self, num):
        """

        ``num``
            A number that should be in the this edge's address range

        return
            ``Point``
                The coordinate of ``num`` within this edge
            Location
                The location, in range [0, 1], of ``num`` within this edge

        """
        # Sanity check; num should always be an `int`
        num = int(num or 0)

        # Determine location in [0, 1] of num along edge
        # Note: addrs might be NULL/None
        addrs = (self.addr_f_l, self.addr_f_r, self.addr_t_l, self.addr_t_r)
        min_addr = min(addrs)
        max_addr = max(addrs)
        if not num or None in (min_addr, max_addr):
            location = .5
        else:
            if min_addr == max_addr:
                location = .5
            else:
                edge_len = max_addr - min_addr
                dist_from_min_addr = num - min_addr
                location = float(dist_from_min_addr) / edge_len

        c = self.c

        # Function to get interpolated point
        f = func.line_interpolate_point(c.geom, location)
        # Function to get WKB version of lat/long point
        f = func.asbinary(f)

        # Query DB and get WKB POINT
        select_ = select([f.label('wkb_point')], c.id == self.id)
        result = select_.execute()
        wkb_point = result.fetchone().wkb_point

        point = geometry.Geometry.fromWKB(wkb_point)
        return point, location

    def splitAtGeocode(self, geocode, node_id=-1, edge_f_id=-1, edge_t_id=-2):
        """Split this edge at ``geocode`` and return two new edges.

        ``geocode`` `Geocode` -- The geocode to split the edge at.

        See `splitAtLocation` for further details

        """
        edge_f, edge_t = self.splitAtLocation(
            geocode.xy, geocode.location, node_id, edge_f_id, edge_t_id
        )
        # address range
        num = geocode.address.number
        edge_f.addr_t_l, edge_f.addr_t_r = num, num
        edge_t.addr_f_l, edge_t.addr_f_r = num, num
        return edge_f, edge_t

    def splitAtNumber(self, num, node_id=-1, edge_f_id=-1, edge_t_id=-2):
        """Split this edge at ``num`` and return two new edges.

        ``num`` `int` -- The address number to split the edge at.

        See `splitAtLocation` for further details

        """
        point, location = self.getPointAndLocationOfNumber(num)
        return self.splitAtLocation(
            point, location, node_id, edge_f_id, edge_t_id
        )

    def splitAtLocation(self, point, location,
                        node_id=-1, edge_f_id=-1, edge_t_id=-2):
        """Split this edge at ``location`` and return two new edges.

        The first edge is `node_f`=>``num``; the second is ``num``=>`node_t`.
        Distribute attributes of original edge to the two new edges.

        ``location`` `float` -- Location in range [0, 1] to split at
        ``node_id`` -- Node ID to assign the node at the split
        ``edge_f_id`` -- Edge ID to assign the `node_f`=>``num`` edge
        ``edge_t_id`` -- Edge ID to assign the ``num``=>`node_t` edge

        return `Edge`, `Edge` -- `node_f`=>``num``, ``num``=>`node_t` edges

        Recipe:
        - Determine location of num along edge; use .5 as default
        - Get XY at location
        - Get line geometry on either side of XY
        - Transfer geometry and attributes to two new edges
        - Return the  two new edges

        """
        num_points = self.geom.numPoints()
        points = [self.geom.pointN(i) for i in range(num_points)]
        N = int(num_points * location) or 1
        if N == num_points:
            N -= 1
        edge_f_points = points[:N] + [point]
        edge_t_points = [point] + points[N:]
        srs = self.geom.srs
        edge_f_geom = geometry.LineString(points=edge_f_points, srs=srs)
        edge_t_geom = geometry.LineString(points=edge_t_points, srs=srs)

        RegionEdge = self.__class__
        edge_f = RegionEdge(id=edge_f_id,
                            node_f_id=self.node_f_id, node_t_id=node_id,
                            street_name=self.street_name,
                            geom=edge_f_geom)
        edge_t = RegionEdge(id=edge_t_id,
                            node_f_id=node_id, node_t_id=self.node_t_id,
                            street_name=self.street_name,
                            geom=edge_t_geom)

        RegionNode = self.node_f.__class__
        geom = self.geom
        shared_node = RegionNode(id=node_id, geom=geom.pointN(N))
        edge_f.node_f = RegionNode(id=self.node_f_id, geom=geom.startPoint())
        edge_f.node_t = shared_node
        edge_t.node_f = shared_node
        edge_t.node_t = RegionNode(id=self.node_t_id, geom=geom.endPoint())

        return edge_f, edge_t

    def __str__(self):
        stuff = [
            joinAttrs(('Address Range:',
                       self.addr_f_l, ', ', self.addr_f_r,
                       'to',
                       self.addr_t_l, ', ', self.addr_t_r)),
            (self.street_name or '[No Street Name]'),
            self.place_l, self.place_r,
        ]
        return joinAttrs(stuff, join_string='\n')

    def to_builtin(self):
        return super(Edge, self).to_builtin()

class StreetName(Entity):
    has_field('prefix', String(2))
    has_field('name', String)
    has_field('sttype', String(4))
    has_field('suffix', String(2))

    def __str__(self):
        attrs = (
            (self.prefix or '').upper(),
            self._name_for_str(),
            (self.sttype or '').title(),
            (self.suffix or '').upper()
        )
        return joinAttrs(attrs)

    def to_builtin(self):
        return {
            'prefix': (self.prefix or '').upper(),
            'name': self._name_for_str(),
            'sttype': (self.sttype or '').title(),
            'suffix': (self.suffix or '').upper()
        }

    def _name_for_str(self):
        """Return lower case name if name starts with int, else title case."""
        name = self.name
        no_name = '[No Street Name]'
        try:
            int(name[0])
        except ValueError:
            name = name.title()
        except TypeError:
            # Street name not set (`None`)
            if name is None:
                name = name = no_name
            else:
                name = str(name)
        except IndexError:
            # Empty street name ('')
            name = no_name
        else:
            name = name.lower()
        return name

    def __nonzero__(self):
        """A `StreetName` must have at least a `name`."""
        return bool(self.name)

    def __eq__(self, other):
        self_attrs = (self.prefix, self.name, self.sttype, self.suffix)
        try:
            other_attrs = (other.prefix, other.name, other.sttype, other.suffix)
        except AttributeError:
            return False
        return (self_attrs == other_attrs)

    def almostEqual(self, other):
        self_attrs = (self.name, self.sttype)
        try:
            other_attrs = (other.name, other.sttype)
        except AttributeError:
            return False
        return (self_attrs == other_attrs)

class City(Entity):
    has_field('city', String)

    def __str__(self):
        if self.city:
            return self.city.title()
        else:
            return '[No City]'

    def to_builtin(self):
        return {
            'id': self.id,
            'city': str(self)
        }

    def __nonzero__(self):
        return bool(self.city)

class State(Entity):
    has_field('code', CHAR(2))  # Two-letter state code
    has_field('state', String)

    def __str__(self):
        if self.code:
            return self.code.upper()
        else:
            return '[No State]'

    def to_builtin(self):
        return {
            'id': self.id,
            'code': str(self),
            'state': str(self.state or '[No State]').title()
        }

    def __nonzero__(self):
        return bool(self.code or self.state)

class Place(Entity):
    has_field('zip_code', Integer)
    belongs_to('city', of_kind='City', **cascade_args)
    belongs_to('state', of_kind='State', **cascade_args)

    def _get_city_name(self):
        return (self.city.city if self.city is not None else None)
    def _set_city_name(self, name):
        if self.city is None:
            self.city = City()
        self.city.city = name
    city_name = property(_get_city_name, _set_city_name)

    def _get_state_code(self):
        return (self.state.code if self.state is not None else None)
    def _set_state_code(self, code):
        if self.state is None:
            self.state = State()
        self.state.code = code
    state_code = property(_get_state_code, _set_state_code)

    def _get_state_name(self):
        return (self.state.state if self.state is not None else None)
    def _set_state_name(self, name):
        if self.state is None:
            self.state = State()
        self.state.state = name
    state_name = property(_get_state_name, _set_state_name)

    def __str__(self):
        city_state = joinAttrs([self.city, self.state], ', ')
        return joinAttrs([city_state, str(self.zip_code or '')])

    def to_builtin(self):
        return {
            'city': (self.city.to_builtin() if self.city is not None else None),
            'state': (self.state.to_builtin() if self.state is not None else None),
            'zip_code': str(self.zip_code or None)
        }

    def __nonzero__(self):
        return bool(self.city or self.state or (self.zip_code is not None))


# type: ignore

import time
import patch_ftp
import robomaster.config  # noqa
from robomaster import robot, logger, logging  # noqa
import robomaster.conn
import robomaster.camera
import cv2
import sys
# import threading
import patch_ftp  # noqa

# IP = "127.0.0.1" # Leave to None to scan for IPs on (0.0.0.0)
IP = ""

run_video = True

def _video(ep_robot):
    print('start video stream')
    ep_robot.camera.start_video_stream(display=False, resolution="720p")
    global run_video
    run_video = True
    while run_video:
        try:
            frame = ep_robot.camera.read_cv2_image()
        except:
            frame = None
        if frame is not None:
            cv2.imshow('image', frame)
            cv2.waitKey(1)
        else:
            time.sleep(0.01)
    ep_robot.camera.stop_video_stream()

def _action(ep_robot):
    global run_video
    print('start action')
    for i in range(12):
        if i % 2:
            r, g, b = 200, 0, 0
            ep_robot.set_robot_mode('free')
            ep_robot.blaster.set_led(effect='off')
        else:
            r, g, b = 0, 0, 200
            ep_robot.set_robot_mode('chassis_lead')
            ep_robot.blaster.set_led(brightness=100, effect='on')
        ep_robot.led.set_led(r=r, g=g, b=b, effect=robomaster.led.EFFECT_BREATH, freq=10)
        ep_robot.chassis.move(x=1, y=0, z=0, xy_speed=1.0, z_speed=90).wait_for_completed()
        ep_robot.chassis.move(x=0, y=0, z=90, xy_speed=1.0, z_speed=90).wait_for_completed()
        ep_robot.gimbal.moveto(yaw=-90, pitch=-20, yaw_speed=90, pitch_speed=30).wait_for_completed()
        ep_robot.gimbal.moveto(yaw=-45, pitch=0, yaw_speed=90, pitch_speed=30).wait_for_completed()
    run_video = False

def main():
    logger.setLevel(logging.WARN)

    if IP:
        robomaster.config.LOCAL_IP_STR = IP
        robomaster.config.ROBOT_IP_STR = IP

    if(len(sys.argv) > 1):
        protocol = sys.argv[-1]
        robomaster.config.ep_conf.video_stream_proto = protocol
    ep_robot = robot.Robot()
    ep_robot.initialize(conn_type="sta")
    # action = threading.Thread(target=_action, args=(ep_robot,))
    # action.start()
    # _video(ep_robot)
    _action(ep_robot)
    ep_robot.close()

if __name__ == '__main__':
    main()


import glob
import json
import cv2
import pytesseract
from pytesseract import Output
from image_preprocessing import ImagePreprocessing

def get_from_d(d, i):
    return d[i]['left'], d[i]['top'], d[i]['width'], d[i]['height'], d[i]['conf'], d[i]['line_num'], d[i]['par_num'], \
           d[i]['block_num'], d[i]['text']

def transform_d(d):
    dd = []
    n_boxes = len(d['left'])
    for i in range(n_boxes):
        (x, y, w, h, c, l, q, b, t) = d['left'][i], d['top'][i], d['width'][i], d['height'][i], d['conf'][i], \
                                      d['line_num'][i], d['par_num'][i], d['block_num'][i], d['text'][i]
        # filter out all elements with width and height less than 5px
        if w <= 5 or h <= 5 or t == '':
            continue

        dd.append({'left': x, 'top': y, 'width': w, 'height': h, 'conf': c, 'line_num': l, 'par_num': q, 'block_num': b,
                   'text': t})

    # join all elements who are closer than 5 px
    i = 0
    while i < len(dd):
        found = True
        while found and i < len(dd):
            found = False
            try:
                (x1, y1, w1, h1, c1, l1, q1, b1, t1) = get_from_d(dd, i)
            except Exception as e:
                print(e)
                continue
            if t1.strip() == '':
                continue
            foundIdx = -1
            foundD = ''
            for j in range(len(dd)):
                (x2, y2, w2, h2, c2, l2, q2, b2, t2) = get_from_d(dd, j)
                if t2.strip() == '':
                    continue
                # if int(c2) <= 0:
                #     continue
                if x1 < x2:
                    d = abs(x2 - x1 - w1)
                    if d < 16:
                        if (y1 <= y2 and y1 + h1 >= y2) or (y2 <= y1 and y2 + h2 >= y1):
                            found = True
                            foundIdx = j
                            if d > 2:
                                foundD = ' '
                            break
            if found:
                (x2, y2, w2, h2, c2, l2, q2, b2, t2) = get_from_d(dd, foundIdx)
                x = min(x1, x2)
                y = min(y1, y2)
                t = t1 + foundD + t2
                mx = max(x1 + w2, x2 + w2)
                my = max(y1 + h1, y2 + h2)
                w = mx - x
                h = my - y
                c = c1
                l = l1
                q = q1
                b = b1
                dd[i] = {'left': x, 'top': y, 'width': w, 'height': h, 'conf': c, 'line_num': l, 'par_num': q,
                         'block_num': b, 'text': t}
                dd.remove(dd[foundIdx])
        i = i + 1
    return dd

def get_coords_one_file(filename, page, preprocess=False):
    coords = []
    config = ''

    # preprocess
    if preprocess:
        img = ImagePreprocessing.preprocess(filename)
    else:
        img = cv2.imread(filename)

    dd = pytesseract.image_to_data(img, output_type=Output.DICT, lang='deu', config=config)
    dd = transform_d(dd)
    for d in dd:
        coords.append({"x_0": d['left'], "y_0": d['top'], "x_1": d['left'] + d['width'], "y_1": d['top'] + d['height'],
                       "page": page, "text": d['text']})
    return coords

def get_coords_file(filename):
    page = 0
    coords = []
    for file in glob.glob(filename + '*.jpg'):
        page = page + 1
        onecoords = get_coords_one_file(file, page)
        for c in onecoords:
            coords.append(c)

    f = open(filename + "_coords.json", "w")
    json.dump(coords, f)
    f.close()


# -*- coding: utf-8 -*-
"""
Created on Sun Oct 17 16:35:09 2021

@author: jkuhnsman
"""

import requests
import re
import boto3
import time
from datetime import datetime as dt
from datetime import timedelta

url = "http://155.138.239.20:7935/metrics"

def getMetrics(url):
    r = requests.get(url, verify=False)
    raw = r.text
    raw_split = raw.split('\n')

    metrics = []

    for m in raw_split:
        if (not '#' in m) & ('livepeer' in m):
            metrics.append(m)

    metrics_parsed = {}

    for m in metrics:
        l = re.split('{|}',m)
        metrics_parsed[l[0]] = float(str(l[-1]).strip())

    return metrics_parsed

#%%

if __name__ == "__main__":
    healthy = True

    while True:
        metrics = getMetrics(url)
        if 'livepeer_transcoders_capacity' in metrics.keys():
            print(metrics['livepeer_transcoders_capacity'])
            if int(metrics['livepeer_transcoders_capacity']) > 0:
                healthy = True
            else:
                healthy = False
        else:
            healthy=False

        f = open("/root/http/index.html", "w")
        if healthy:
            f.write("healthy - frankfurt orch")
        else:
            f.write("orch is down - frankfurt orch")
        f.close()

        time.sleep(5)


from selenium import webdriver
import time

try:
    browser = webdriver.Chrome()
    # 需要安装chrome driver, 和浏览器版本保持一致
    # http://chromedriver.storage.googleapis.com/index.html

    browser.get('https://shimo.im')
    time.sleep(1)
    # browser.switch_to_frame(browser.find_elements_by_tag_name('iframe')[0])
    btm1 = browser.find_element_by_xpath('/html/body/div[1]/div[1]/header/nav/div[3]/a[2]/button')
    btm1.click()
    time.sleep(5)
    browser.find_element_by_xpath('//*[@id="root"]/div/div[2]/div/div/div/div[2]/div/div/div[1]/div[1]/div/input').send_keys('15055495@qq.com')
    browser.find_element_by_xpath('//*[@id="root"]/div/div[2]/div/div/div/div[2]/div/div/div[1]/div[2]/div/input').send_keys('test123test456')
    time.sleep(3)
    browser.find_element_by_xpath('//*[@id="root"]/div/div[2]/div/div/div/div[2]/div/div/div[1]/button').click()

    cookies = browser.get_cookies() # 获取cookies
    print(cookies)
    time.sleep(3)

except Exception as e:
    print(e)
finally:
    time.sleep(10)
    browser.close()


import threading

from multiprocessing import Queue

class UpstreamThread(threading.Thread):
    def __init__(self):
        threading.Thread.__init__(self, daemon=True)
        self.queue = Queue()
        self.socket = None
        self.socket_lock = threading.RLock()
        self.running = True

    def set_socket(self, socket):
        self.clear()
        with self.socket_lock:
            self.socket = socket

    def connected(self):
        with self.socket_lock:
            return self.socket is not None

    def put(self, b):
        self.queue.put(b)

    def clear(self):
        while not self.queue.empty():
            self.queue.get()

    def stop(self):
        self.set_socket(None)
        self.running = False

    def run(self):
        while self.running:
            if not self.queue.empty():
                # Acquire the lock since socket can be None when set in another thread
                with self.socket_lock:
                    if self.socket:
                        while not self.queue.empty():
                            pkt = self.queue.get()
                            try:
                                self.socket.send(pkt)
                            except Exception as _:
                                pass # Keep on throwing exceptions until we get a new socket


from django.db import models

# Create your models here.

class Employee(models.Model):
    CATEGORY = [
        ('Java', 'Java'),
        ('Python', 'Python'),
        ('.NET', '.NET'),
        ('Testing', 'Testing'),
        ('Salesforce', 'Salesforce'),
        ('Deployment', 'Deployment'),
        ('DevOps', 'DevOps'),
        ('Cybersecurity', 'Cybersecurity'),
        ('DataScience', 'DataScience')
    ]

    name = models.CharField(max_length=50)
    age = models.IntegerField()
    skills = models.CharField(max_length=20, choices=CATEGORY)

    def __str__(self):
        return self.name


from admin_loader import dp, pay_bot
from aiogram import types
from aiogram.dispatcher.filters.builtin import Command
from aiogram.utils.markdown import hlink
from data.config import ADMINS
import datetime
from dateutil import relativedelta
import logging
from utils.db_api.db_commands import database

@dp.message_handler(Command("kickusers"), state="*")
async def bot_kickusers(message: types.Message = None):
    """
    /kickusers функция бота.
    """

    t1 = datetime.datetime.now()
    text = "Start kickChatMember method\n"
    if message is not None:
        await message.answer(text=text)

    data = await database.get_users_active_subscribes()
    time = datetime.datetime.now()
    N = 0
    for row in data:
        if row["paid_till"] < time:
            channel_id = int(row["subscribe_channel_id"])
            chat_id = int(row["chat_id"])
            await pay_bot.kick_chat_member(channel_id, chat_id)
            await pay_bot.unban_chat_member(channel_id, chat_id)
            await database.update_user_subscribe_status(
                subscribe_channel_id=channel_id,
                chat_id=chat_id,
                subscribe_status="Expired",
                subscribe_status_previous="Active")
            N += 1
        elif row["paid_till"] < time + relativedelta.relativedelta(
                days=+2) and row["last_notify_time"] < time - relativedelta.relativedelta(days=+1):
            channel_id = int(str(row["subscribe_channel_id"])[4:])
            chat_id = int(row["chat_id"])
            text = "У вас заканчивается подписка на канал!\n"
            text += "Канал "
            text += hlink(f"{row['subscribe_channel']}\n",
                          f"https://t.me/c/{channel_id}/1")
            text += f"Срок подписки до {row['paid_till']}\n"
            try:
                await pay_bot.send_message(chat_id, text)
                await database.update_user_subscribe_last_notify_time(
                    last_notify_time=time,
                    subscribe_id=row["id"])
            except Exception as err:
                logging.exception(err)

    t2 = datetime.datetime.now()
    text = "Finish kickChatMember method\n"
    text += f"Kick N={N} users\n"
    time = t2 - t1
    text += f"Time: {time}\n"
    if message is not None:
        await message.answer(text=text)
    elif N > 0:
        for admin in ADMINS:
            await pay_bot.send_message(admin, text)


# -*- coding:utf-8 -*-
"""
@name:   maoyan.py
@time:   2019 / 03 / 24
@author: taifu
"""

import json
import requests
from requests.exceptions import RequestException
import re

# 获取网页源文件
def get_one_page(url):
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        return None

# 单页解析
def parse_one_page(html):
    pattern = re.compile('
.*?board-index.*?>(\d+).*?data-src="(.*?)".*?name.*?>(.*?).*?star">(.*?)
.*?releasetime">(.*?)
.*?integer">(.*?).*?fraction">(.*?)', re.S)
    items = re.findall(pattern, html)
    for item in items:
        yield {
            'index': item[0],
            'image': item[1],
            'title': item[2],
            'actor': item[3].strip()[3:],
            'time': item[4].strip()[5:],
            'score': item[5]+item[6]
        }

# 保存到文件
def write_to_file(content):
    with open('maoyan.txt', 'a', encoding='utf-8') as f:
        f.write(json.dumps(content, ensure_ascii=False) + '\n')
        f.close()

# 爬取
def crawl(offset):
    url = 'https://maoyan.com/board/4?offset=' + str(offset)
    html = get_one_page(url)
    for item in parse_one_page(html):
        print(item)
        write_to_file(item)

# 入口
if __name__ == '__main__':
    for i in range(10):
        crawl(i*10)


# -*- coding: utf-8 -*-
# @Project:AID1810
# @Author:biabu
# @Date:2019/4/15 11:33
# @File_name:02_keywordExtraction.py
# @IDE:PyCharm

"""
贝叶斯新闻关键词提取 和 新闻分类
"""

import pandas as pd
import jieba
import jieba.analyse
from wordcloud import WordCloud
import matplotlib.pyplot as plt
import matplotlib
import numpy as np
from gensim import corpora
import gensim
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB

class KeywordExtraction():
    """
    ---------------贝叶斯新闻关键词提取和分类----------------
    """

    def __init__(self):
        """读取数据"""
        data = pd.read_table('./data/val.txt', names=['category', 'theme', 'URL', 'content'], encoding='utf-8')
        self.data = data.dropna()
        self.stopwords = pd.read_csv("./data/stopwords.txt",index_col=False,sep="\t",quoting=3,names=['stopword'], encoding='utf-8')

    def content(self):
        """
        使用结巴分词器对新闻内容进行分词
        :return: 返回分词好的新闻内容列表
        """
        content = self.data.content.values.tolist()
        content_S = []
        for line in content:
            current_segment = jieba.lcut(line)
            if len(current_segment) > 1 and current_segment is not '\n\r':
                content_S.append(current_segment)
        return content_S

    def drop_stopwords(self):
        """
        从新闻内容中删除掉停用词，留下基本关键词
        :return:  list of list 格式的基本关键词，所有的词统计
        """
        contents = self.content()
        stopwords = self.stopwords.stopword.values.tolist()

        contents_clean = []
        all_words = []
        for line in contents:
            line_clean = []  # 每篇新闻内容的基本关键词
            for word in line:
                if word in stopwords:
                    continue
                line_clean.append(word)
                all_words.append(str(word))
            contents_clean.append(line_clean)
        return contents_clean, all_words

    def word_cloud_plot(self, all_words):
        """
        根据基本关键词，进行词云图绘制
        :return: None
        """
        df_words = pd.DataFrame({'all_words': all_words})  # 转换基本关键词格式
        words_count = df_words.groupby(by=['all_words'])['all_words'].agg({'count': np.size})  # 进行分组聚合
        words_count = words_count.reset_index().sort_values(by=["count"], ascending=False)  # 排序重新设置索引

        matplotlib.rcParams['figure.figsize'] = (10.0, 5.0)
        wordcloud = WordCloud(font_path='./data/simhei.ttf', background_color='white', max_font_size=80)
        word_frequence = {x[0]: x[1] for x in words_count.head(100).values}
        wordcloud = wordcloud.fit_words(word_frequence)
        plt.imshow(wordcloud)
        plt.show()

    def tf_idf(self, contents_clean):
        """
        TF-IDF提取关键词, LDA 主体模型
        :return:
        """
        # gensim 获取全局集合字典
        dic = corpora.Dictionary(contents_clean)
        corpus = [dic.doc2bow(sentence) for sentence in contents_clean]  # 获取语料库
        lda = gensim.models.ldamodel.LdaModel(corpus=corpus, id2word=dic, num_topics=20)  # lda 线性判别分析
        topics = []
        for topic in lda.print_topics(num_topics=20,num_words=5):
            topics.append(topic)
        topics = pd.DataFrame({'topics': topics})
        print(topics.head())

    def jieba_analyze(self):
        """
        结巴分词关键词分析
        :return: 文章关键词
        """
        content_S = self.content()
        keywords = []
        for i in range(len(content_S)):
            content_S_str = ''.join(content_S[i])
            keyword = ' '.join(jieba.analyse.extract_tags(content_S_str, topK=5, withWeight=False))
            keywords.append(keyword)

        self.data['keywords'] = keywords
        print(self.data.head())
        # return keywords

    def classify(self, contents_clean):
        """
        新闻分类
        :return:
        """
        df_train = pd.DataFrame({'contents_clean':contents_clean,'label':self.data['category']})
        # 分类数字化
        label_mapping = {"汽车": 1, "财经": 2, "科技": 3, "健康": 4, "体育": 5, "教育": 6, "文化": 7, "军事": 8, "娱乐": 9, "时尚": 0}
        df_train['label'] = df_train['label'].map(label_mapping)  # 映射标签

        # 切分数据
        x_train, x_test, y_train, y_test = train_test_split(df_train['contents_clean'].values, df_train['label'].values,
                                                            test_size=0.25)
        # x_train, content 统计转换向量化
        words = []
        for line_index in range(len(x_train)):
            try:
                words.append(' '.join(x_train[line_index]))
            except:
                continue
        cv = CountVectorizer(analyzer='word', max_features=4000, lowercase=False)
        x_train = cv.fit_transform(words)

        # x_test, content 统计转换向量化
        test_words = []
        for line_index in range(len(x_test)):
            try:
                test_words.append(' '.join(x_test[line_index]))
            except:
                continue
        x_test = cv.transform(test_words)

        # 构建贝叶斯分类器, 利用统计向量转换进行分析
        mlt = MultinomialNB()
        mlt.fit(x_train, y_train)
        print(mlt.score(x_test, y_test))

        # 中文字符特征化, 判别分词的重要性
        tfidf = TfidfVectorizer(analyzer='word', max_features=4000, lowercase=False)
        x_train = tfidf.fit_transform(words)
        x_test = tfidf.transform(test_words)

        # 构建贝叶斯分类器， 利用分词进行分析
        mlt = MultinomialNB()
        mlt.fit(x_train, y_train)
        print(mlt.score(x_test, y_test))

    def workOn(self):
        """ 启动函数"""
        contents_clean, all_words = self.drop_stopwords()
        # self.word_cloud_plot(all_words)
        # self.tf_idf(contents_clean)
        self.classify(contents_clean)

if __name__ == '__main__':
    ke = KeywordExtraction()
    ke.workOn()
    # ke.jieba_analyze()


"""Unit tests for ``:mod:zarp.plugins.sample_processors.genomepy``."""

import logging
from pathlib import Path
import shutil
from unittest.mock import Mock

from genomepy.genome import Genome
import numpy as np
import pandas as pd

from zarp.config.models import Config, ConfigRun, ConfigSample, ConfigUser
from zarp.samples.sample_record_processor import SampleRecordProcessor as SRP
from zarp.plugins.sample_processors.genomepy import (
    SampleProcessorGenomePy as SPG,
)

LOGGER = logging.getLogger(__name__)

class TestSampleProcessorGenomePy:
    """Test ``cls:zarp.plugins.sample_processors.SampleProcessorGenomePy``."""

    config = Config(
        run=ConfigRun(
            zarp_directory=Path(__file__).parents[2] / "files" / "zarp",
            genome_assemblies_map=Path(__file__).parents[2]
            / "files"
            / "genome_assemblies.csv",
        ),
        sample=ConfigSample(fragment_length_distribution_mean=1000),
        user=ConfigUser(),
    )
    data = pd.DataFrame(
        data={
            "identifier": ["sample1", "sample2"],
            "type": ["LOCAL_LIB_SINGLE", "LOCAL_LIB_SINGLE"],
            "orientation": ["SF", "SF"],
            "source": ["Schizosaccharomyces pombe", 4932],
        }
    )
    assemblies = ["ASM294v2", "R64-1-1"]

    def test_init(self):
        """Test constructor."""
        config = self.config.copy(deep=True)
        df = self.data.copy(deep=True)
        srp = SRP()
        srp.append(df)
        spg = SPG(config=config, records=srp.records)
        assert hasattr(spg, "records")
        assert len(spg.records.index) == 2

    def test_process(self, monkeypatch, tmpdir):
        """Test `.process()` method."""
        config = self.config.copy(deep=True)
        df = self.data.copy(deep=True)
        srp = SRP()
        srp.append(df)
        spg = SPG(config=config, records=srp.records)
        genomes_dir_root = Path(tmpdir)
        return_data = []
        for assembly in self.assemblies:
            genomes_dir = genomes_dir_root / "latest" / assembly
            genomes_dir.mkdir(parents=True, exist_ok=False)
            shutil.copyfile(
                Path(__file__).parents[2] / "files" / "fasta",
                genomes_dir / f"{assembly}.fa",
            )
            shutil.copyfile(
                Path(__file__).parents[2] / "files" / "gtf",
                genomes_dir / f"{assembly}.annotation.gtf",
            )
            obj = Genome(assembly, genomes_dir=genomes_dir_root / "latest")
            obj.annotation_gtf_file = (
                genomes_dir / f"{assembly}.annotation.gtf"
            )
            return_data.append(obj)
        monkeypatch.setattr(
            "genomepy.install_genome",
            Mock(side_effect=return_data),
        )
        df_out = spg.process()
        assert len(df_out.index) == 2
        assert (
            df_out["reference_sequences"].iloc[0]
            == genomes_dir_root
            / "latest"
            / self.assemblies[0]
            / f"{self.assemblies[0]}.fa"
        )
        assert (
            df_out["annotations"].iloc[0]
            == genomes_dir_root
            / "latest"
            / self.assemblies[0]
            / f"{self.assemblies[0]}.annotation.gtf"
        )
        assert (
            df_out["reference_sequences"].iloc[1]
            == genomes_dir_root
            / "latest"
            / self.assemblies[1]
            / f"{self.assemblies[1]}.fa"
        )
        assert (
            df_out["annotations"].iloc[1]
            == genomes_dir_root
            / "latest"
            / self.assemblies[1]
            / f"{self.assemblies[1]}.annotation.gtf"
        )

    def test_process_empty(self, caplog):
        """Test `.process()` method with no records."""
        config = self.config.copy(deep=True)
        srp = SRP()
        srdd = SPG(config=config, records=srp.records)
        assert len(srdd.records.index) == 0
        with caplog.at_level(logging.DEBUG):
            df_out = srdd.process()
        assert len(df_out.index) == 0
        assert "No genome resources to fetch" in caplog.text

    def test_set_assemblies(self):
        """Test `.set_assemblies()` method."""
        config = self.config.copy(deep=True)
        df = self.data.copy(deep=True)
        srp = SRP()
        srp.append(df)
        spg = SPG(config=config, records=srp.records)
        spg.set_assemblies()
        assert "assembly" in spg.records.columns
        assert len(spg.records["assembly"]) == 2
        assert list(spg.records["assembly"]) == ["ASM294v2", "R64-1-1"]

    def test_fetch_resources(self, monkeypatch, tmpdir):
        """Test `.fetch_resources()` method."""
        config = self.config.copy(deep=True)
        config.run.resources_version = 50
        df = self.data.copy(deep=True)
        srp = SRP()
        srp.append(df)
        spg = SPG(config=config, records=srp.records)
        spg.set_assemblies()
        genomes_dir_root = Path(tmpdir)
        return_data = []
        for assembly in self.assemblies:
            genomes_dir = (
                genomes_dir_root / str(config.run.resources_version) / assembly
            )
            genomes_dir.mkdir(parents=True, exist_ok=False)
            shutil.copyfile(
                Path(__file__).parents[2] / "files" / "fasta",
                genomes_dir / f"{assembly}.fa",
            )
            shutil.copyfile(
                Path(__file__).parents[2] / "files" / "gtf",
                genomes_dir / f"{assembly}.annotation.gtf",
            )
            obj = Genome(
                assembly,
                genomes_dir=genomes_dir_root
                / str(config.run.resources_version),
            )
            obj.annotation_gtf_file = (
                genomes_dir / f"{assembly}.annotation.gtf"
            )
            return_data.append(obj)
        # test with genome files and annotation files available
        monkeypatch.setattr(
            "genomepy.install_genome",
            Mock(side_effect=return_data),
        )
        paths = spg.fetch_resources(genomes_dir_root=genomes_dir_root)
        assert len(paths) == 2
        # test with one genome file and one annotation file missing
        Path(paths[self.assemblies[0]][0]).unlink()
        Path(paths[self.assemblies[1]][1]).unlink()
        monkeypatch.setattr(
            "genomepy.install_genome",
            Mock(side_effect=return_data),
        )
        paths = spg.fetch_resources(genomes_dir_root=genomes_dir_root)
        assert len(paths) == 0

    def test_set_resource_paths(self):
        """Test `.set_resource_paths()` method."""
        config = self.config.copy(deep=True)
        df = self.data.copy(deep=True)
        srp = SRP()
        srp.append(df)
        spg = SPG(config=config, records=srp.records)
        spg.set_assemblies()
        resource_paths = dict(
            zip(self.assemblies, [(Path(), Path()), (Path(), Path())])
        )
        # test when for all records, assemblies are available
        spg.set_resource_paths(resource_paths=resource_paths)
        assert spg.records["reference_sequences"].iloc[0] == Path()
        assert spg.records["annotations"].iloc[0] == Path()
        assert spg.records["reference_sequences"].iloc[1] == Path()
        assert spg.records["annotations"].iloc[1] == Path()
        # test when for one record, assembly is not available
        spg.records["assembly"].iloc[0] = None
        spg.set_resource_paths(resource_paths=resource_paths)
        assert np.isnan(spg.records["reference_sequences"].iloc[0])
        assert np.isnan(spg.records["annotations"].iloc[0])
        assert spg.records["reference_sequences"].iloc[1] == Path()
        assert spg.records["annotations"].iloc[1] == Path()


import requests
from django.http import HttpRequest, HttpResponse, JsonResponse
from django.shortcuts import render, redirect
from django.contrib import messages
from django.views.generic import DetailView, ListView, TemplateView
from django.views.generic.edit import FormMixin
from django.contrib.auth.decorators import login_required
from django.views.decorators.csrf import csrf_exempt

from .models import Property, PropertyType

from inquiry.forms import InquiryForm

from search.models import SearchCriteria

from .provinces import provinces

# Create your views here.

class Home(ListView):
    queryset = Property.objects.filter(active=True, deal=False)
    model = Property
    template_name = 'home.html'

    def get_context_data(self, **kwargs):
        context = super(Home, self).get_context_data(**kwargs)
        locations = list(Property.objects.filter(active=True, deal=False).values())
        context['locations'] = locations

        return context

class About(TemplateView):
    template_name = 'about.html'

def search_ajax(request):
    if 'term' in request.GET:
        q = request.GET.get('term', '').capitalize()
        ps = list()
        for p in provinces:
            if q in p:
                ps.append(p)

    return JsonResponse(ps, safe=False)

class PropertyListView(ListView):
    model = Property
    template_name = 'property/property_list.html'

    def get_context_data(self, **kwargs):
        context = super(PropertyListView, self).get_context_data(**kwargs)
        global properties
        properties =  Property.objects.filter(active=True, deal=False)
        context['properties'] = properties
        context['page_title'] = 'Properties'
        context['message'] = 'Nothing to show'
        return context

    def post(self, request, *args, **kwargs):
        global properties
        form = self.request.POST

        if 'search' in form:
            properties = Property.objects.filter(active=True, deal=False)
            if form['type'] != '':
                properties = properties.filter(type__title=form['type'])
            if form['city']:
                properties = properties.filter(city__icontains=form['city'])
            if form['price_from'] and form['price_to']:
                properties = properties.filter(price__range=(int(form['price_from']), int(form['price_to'])))
            if form['floors']:
                properties = properties.filter(floor=form['floors'])
            if form['bedrooms']:
                properties = properties.filter(bedroom=form['bedrooms'])
            if form['bathrooms']:
                properties = properties.filter(bathroom=form['bathrooms'])
            if form['parking']:
                properties = properties.filter(parking=form['parking'])

            if form.get('basement'):
                properties = properties.filter(basement=True)
            if request.user.is_authenticated:
                search = SearchCriteria.objects.filter(user=request.user).first()
                if not search:
                    search = SearchCriteria.objects.create(user=request.user)
                property_type = PropertyType.objects.filter(title=form['type']).first()
                if property_type:
                    search.type = property_type
                else:
                    search.type = None
                search.city = form['city']
                search.price_from = form['price_from']
                search.price_to = form['price_to']
                search.floors = form['floors']
                search.bedrooms = form['bedrooms']
                search.bathrooms = form['bathrooms']
                search.parking = form['parking']
                search.basement = True if form.get('basement') == 'on' else False
                search.save()
            context ={
                'properties': properties,
                'page_title': 'Search',
                'message': 'Nothing found'
            }
            return render(request, 'property/property_list.html', context)

        elif form['type'] == 'filter':
            properties = Property.objects.filter(active=True, deal=False).filter(type__title=form['filter'])
        else:
            properties = properties.order_by(form['sort'])

        context = {
            'properties': properties,
        }

        return render(request, 'property/property_list.html', context)

class PropertyDetailView(FormMixin, DetailView):
    model = Property
    template_name = 'property/property_detail.html'
    context_object_name = 'property'
    form_class = InquiryForm

    def post(self, request, *args, **kwargs):
        form = self.get_form()
        if form.is_valid():
            inquiry = form.save(commit=False)
            inquiry.property = self.get_object()
            inquiry.save()
            messages.success(request, 'Your inquiry has been sent successfully')
        else: messages.warning(request, 'Your inquiry has not been sent successfully')
        return redirect('property:property_detail', pk=self.get_object().id)

class TypePropertyListView(ListView):
    model = Property
    template_name = 'property/property_list.html'
    context_object_name = 'properties'

    def get_queryset(self):
        global properties
        properties = super(TypePropertyListView, self).get_queryset()
        slug = self.kwargs.get('slug')
        properties = Property.objects.filter(type__slug=slug, active=True, deal=False)
        return properties

    def get_context_data(self, **kwargs):
        context = super(TypePropertyListView, self).get_context_data(**kwargs)
        global slug
        slug = self.kwargs.get('slug')
        context['page_title'] = slug.capitalize()
        return context

    def post(self, request, *args, **kwargs):
        global properties
        form = self.request.POST
        if form['type'] == 'filter':
            properties = Property.objects.filter(active=True, deal=False).filter(type__title=form['filter'])
        else:
            properties = properties.order_by(form['sort'])
        context = {
            'properties': properties,
            'page_title': slug.capitalize(),
        }
        return render(request, 'property/property_list.html', context)

@login_required
@csrf_exempt
def add_to_favorites(request):
    form = request.POST
    if not form.get('property_id'):
        print('something is not a valid form')
    else:
        property = Property.objects.filter(id=form.get('property_id')).first()
        if not property:
            print('property not found')
        else:
            request.user.favorites.add(property)
            request.user.save()
    return redirect('property:property_detail', pk=form.get('property_id'))


"""Client for HTTP Requests

This module provides a Client object to interface with an API.

"""

import datetime
import json
import logging
import os
import requests
import time

from apps.apihandler.errors import UnprocessableEntity
from requests import HTTPError
from requests import Session
from requests.adapters import HTTPAdapter
#from requests.compat import urljoin
from requests.packages.urllib3.util.retry import Retry # type: ignore

basedir     = os.path.abspath(os.path.dirname(__file__))
logger      = logging.getLogger(__name__)

'''
/*
 *Urls for asking questions of the documents, retrieving the
 * answers, and giving feedback as to the goodness of references.
 */'''
class Client(Session):
    """GeneNetworkQA Client

    Constructs a :obj:`requests.Session` for GeneNetworkQA API requests with
    authorization, base URL, request timeouts, and request retries.

    Args:
        account (str): base address subdomain
        api_key (str): API key
        version (str, optional): API version, defaults to "v3"
        timeout (int, optional): :obj:`` timeout value, defaults to 5
        total (int, optioanl): :obj:`Retry` total value, defaults to 5
        backoff_factor (int, optional): :obj:`Retry` backoff_factor value, defaults to 30

    Usage::
        from genenetworkqa import Client
        gnqa = Client(account="account-name", api_key="XXXXXXXXXXXXXXXXXXX...")
    """

    def __init__(self, account, api_key, v="v3", timeout=5, total=5, backoff_factor=30) -> None:
        """
        """
        super().__init__()
        self.host = f"https://genenetwork.fahamuai.com/api/tasks"
        self.headers.update(self.getAuth(self.openAPIConfig()))
        self.baseUrl      = 'https://genenetwork.fahamuai.com/api/tasks'
        self.answerUrl    = self.baseUrl + '/answers'
        self.feedbackUrl  = self.baseUrl + '/feedback'
        adapter = TimeoutHTTPAdapter(
            timeout=timeout,
            max_retries=Retry(
                total=total,
                status_forcelist=[429, 500, 502, 503, 504],
                backoff_factor=backoff_factor,
            ),
        )
        self.mount("https://", adapter)
        self.mount("http://", adapter)

    def openAPIConfig(self):
        f = open(os.path.join(basedir, "api.config.json") , "rb" )
        result = json.load(f)
        f.close()
        return result

    def getAuth(self, api_config):
        return {"Authorization": "Bearer " + api_config['Bearer Token October 2023']}

    def ask(self, exUrl, *args, **kwargs):
        askUrl = self.baseUrl + exUrl
        res    = self.custom_request('POST', askUrl, *args, **kwargs)
        if (res.status_code != 200):
            return self.negativeStatusMsg(res), 0
        task_id = self.getTaskIDFromResult(res)
        return res, task_id

    def getAnswer(self, taskid, *args, **kwargs):
        query = self.answerUrl + self.extendTaskID(taskid)
        res   = self.custom_request('GET', query, *args, **kwargs)
        if (res.status_code != 200):
            return self.negativeStatusMsg(res), 0
        return res, 1

    def negativeStatusMsg(self, res):
        return 'Problems\n\tStatus code => {0}\n\tReason=> {res.reason}'.format(res.status_code, res.reason)

    def extendTaskID(self, task_id):
        return '?task_id=' + str(task_id['task_id'])

    def getTaskIDFromResult(self, res):
        return json.loads(res.text)

    def custom_request(self, method, url, *args, **kwargs):
        max_retries = 5
        retry_delay = 3

        print ('[{0}] Request begin'.format(datetime.datetime.now()))
        response = super().request(method, url, *args, **kwargs)
        print ('[{0}] Response arrival'.format(datetime.datetime.now()))

        i = 0
        for i in range(max_retries):
            try:
                print ('Raising status for response {0} -- {1}'.format(i+1, datetime.datetime.now()))
                response.raise_for_status()
            except requests.exceptions.RequestException as exc:
                code = exc.response.status_code
                if code == 422:
                    raise UnprocessableEntity(exc.request, exc.response)
                    #from exc
                elif i == max_retries - 1:
                    raise
            if response.ok:
                print('Status code for response is {0}'.format(response.status_code))
                # Give time to get all the data
                print ('[{0}] delay begin'.format(datetime.datetime.now()))
                time.sleep(retry_delay*3)
                print ('[{0}] delay   end'.format(datetime.datetime.now()))
                return response
            else:
                print ('[{1}] Retry {0}'.format(i+1, datetime.datetime.now()))
                time.sleep(retry_delay)
        return response

class TimeoutHTTPAdapter(HTTPAdapter):
    def __init__(self, timeout, *args, **kwargs):
        """TimeoutHTTPAdapter constructor.

        Args:
            timeout (int): How many seconds to wait for the server to send data before
                giving up.
        """
        self.timeout = timeout
        super().__init__(*args, **kwargs)

    def send(self, request, **kwargs):
        """Override :obj:`HTTPAdapter` send method to add a default timeout."""
        timeout = kwargs.get("timeout")
        if timeout is None:
            kwargs["timeout"] = self.timeout

        return super().send(request, **kwargs)


from rest_framework import serializers
import datetime
from persons.models import Person

'''
#Serializers can convert ---> Json
#Serializers can validate data
'''
#class PersonCustomSerilizer(serializers.Serializer):
#    text_field = serializers.CharField()

class PersonSerializer(serializers.ModelSerializer):
    class Meta:
        model = Person
        fields = [
            'id',
            'Pid',
            'PersonName',
            'Person_sex',
            'Person_BDate',
            'Person_CDate'
        ]

    def validate_personName(self,value):
        if len(value) > 30:
            raise serializers.ValidationError("PersonName should not be more than 30 Char")
        return value
    def validate_personsex(self,value):
        if value not in ['M','F','N']:
            raise  serializers.ValidationError("PersonSex should be M F or N")
        return value
    def validate_personbdate(self,value):
        if datetime.datetime.strptime(value, '%Y-%m-%d'):
            raise serializers.ValidationError("Incorrect data format, should be YYYY-MM-DD")
        return value


import numpy as np
import policy_value_net
import time
import game
from multiprocessing.dummy import Pool as ThreadPool

"""
    蒙特卡洛树搜索
    用于自我对弈和人机对弈
    自我对弈：自我对弈中再下每一步之前将当前局面喂进神经网络，得出当前局面的胜率和接下来每一步的概率，然后进行扩展和backup 以上步骤执行1600次以后
    输出下一步的位置
    然后根据子节点访问的次数进行softmax函数的概率转换
    最后根据加了狄利克雷噪声的概率分布随机取一个值作为下一步
"""

# 搜索树节点 每个节点包含其父子几点 这个节点的先验概率 访问次数 Q值 和 W值
class Node(object):
    def __init__(self, p, parent):
        self.p = p
        self.parent = parent
        self.N = 0
        self.Q = 0
        self.W = 0
        self.children = {}

    # 根据uct值选择值最大的子节点
    def select(self):
        return max(self.children.items(), key=lambda c: c[1].uct())

    # uct值
    def uct(self, c_puct=5.0):
        return self.Q + c_puct * self.p * (np.sqrt(self.parent.N)) / (self.N + 1)

    # 扩展节点
    def expand(self, act_probs):
        for act, prob in act_probs:
            if act not in self.children:
                self.children[act] = Node(prob, self)

    # 沿着扩展到叶子节点的路线回溯将边的统计数据更新
    def backup(self, leaf_value):
        self.W += leaf_value
        self.N += 1
        self.Q = self.W / self.N

    # 判断是否为叶子节点
    def is_leaf(self):
        return len(self.children) == 0

    def update_value(self, leaf_value):
        if self.parent:
            self.parent.update_value(-leaf_value)
        self.backup(leaf_value)

# 蒙特卡洛树
class MCTS(object):
    def __init__(self, net):
        self.net = net
        self.board = None
        self.root = None
        self.mul_process = 16
        self.pool = ThreadPool(self.mul_process)
        self.arg = [k for k in range(0, 400, 400 // self.mul_process)]

    def search(self, board, node, temp=1e-3):
        self.root = node
        self.board = board
        # self.reset_pool()
        # s_time = time.time()
        for _ in range(400):
            node = self.root
            board = self.board.clone()
            while not node.is_leaf():
                action, node = node.select()
                board.do_move(action)
            act_probs, value = self.net.get_value_policy(board)
            end, winner = board.game_end()
            if not end:
                node.expand(act_probs)
            else:
                if winner == -1:
                    value = 0.0
                else:
                    value = 1.0 if winner == board.current_player else -1.0
            node.update_value(-value)
        # for _ in self.arg:
        #     self.pool.apply_async(self.mc, args=())
        # self.pool.close()
        # self.pool.join()
        actions = []
        times = []
        for action, child in self.root.children.items():
            actions.append(action)
            times.append(child.N + 1e-10)
        action, pi = self.decision(actions, times, temp)
        for act, child in self.root.children.items():
            if action == act:
                # e_time = time.time()
                # print(f"Monte Carlo Tree Search takes {(e_time - s_time)} seconds")
                return action, child, pi

    def reset_pool(self):
        self.pool = ThreadPool(self.mul_process)

    @staticmethod
    def dirichlet_noise(probs, eps=0.25, alpha=0.1):
        return (1 - eps) * probs + eps * np.random.dirichlet(np.full(len(probs), alpha))

    def decision(self, acts, pi, temperature):
        pi = (1.0 / temperature) * np.log(pi)
        pi = np.exp(pi - np.max(pi))
        pi /= np.sum(pi)
        # pi = self.dirichlet_noise(pi)
        action = np.random.choice(acts, p=pi)
        return action, pi

# if __name__ == '__main__':
#     net = policy_value_net.Net()
#     b = game.Board()
#     act_probs, value = net.get_value_policy(b)
#     act, probs = map(np.array, zip(*act_probs))
#     n = Node(None, None)
#     mc = MCTS(net)
#     for i in range(100):
#         b = game.Board()
#         n = Node(None, None)
#         mc.search(b, n)


# -*- coding: utf-8 -*-
"""
Created on Mon May  6 06:58:45 2019
Initial build of genre prediction model.
Parses

@author: night
"""
import pandas as pd
import dill
import numpy as np
#from wordcloud import WordCloud, STOPWORDS

from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.model_selection import GridSearchCV
from sklearn import base

from sklearn.naive_bayes import MultinomialNB
from sklearn.naive_bayes import ComplementNB
from sklearn.svm import SVC

from sklearn.neighbors import NearestNeighbors

class RowRemoveTransformer(base.BaseEstimator, base.TransformerMixin):

    def __init__(self, min_count=0):
        self.min_count = min_count  # We will need these in transform()

    def fit(self, X, y=None):
        # This transformer doesn't need to learn anything about the data,
        # so it can just return self without any further processing
        return self

    def transform(self, X):
        # Return an array with the same number of rows as X and one
        # column for each in self.col_names
        return [row for row in X if np.sum(row) > self.min_count]

def save_pkl(df, filename):
    with open('data/'+filename+'.pkl','wb') as fobj:
        dill.dump(df,fobj)

def load_pkl(filename):
    with open('data/'+filename+'.pkl','rb') as fobj:
        df = dill.load(fobj)
    return df

def generate_model(data):

    data = data.sample(frac=1).reset_index(drop=True)

    rock_sample = data[data['genre']=='rock'].sample(3300)
    country_sample = data[data['genre']=='country'].sample(3300)
    hiphop_sample = data[data['genre']=='hiphop'].sample(3300)

    sampled_data = pd.concat([rock_sample, hiphop_sample, country_sample])
    sampled_data = data
    X = list(sampled_data.lyrics)
    y = list(sampled_data.genre)
    stop_words = frozenset(["verse", "chorus","instrumental"])

    lyric_class = Pipeline([
        ('ctvect', CountVectorizer(min_df=0, max_df=0.95, stop_words=stop_words, ngram_range=(1,2))), #Vectorizer
        ('tfidf_trans', TfidfTransformer()), #Normalization
#        ('cnb', ComplementNB(alpha=0.1))
        ('mnb', MultinomialNB())
#        ('svc', SVC(probability=True))
    ])

#    parameters = {'mnb__alpha': [0.01, 0.03, 0.05, 0.08], 'ctvect__min_df': [0.1, 0.5, 0.02], 'ctvect__max_df': [0.99]}
    parameters = {'ctvect__min_df': [0, 0.01], 'ctvect__max_df': [0.8, 0.96, 0.99], 'mnb__alpha': [0.01, 0.1, 1]}
    gscv = GridSearchCV(lyric_class, parameters, cv=4, scoring='balanced_accuracy')
    gscv.fit(X, y)
    print(gscv.best_params_)
    print(gscv.best_score_)

    lyric_clf = gscv.best_estimator_

    save_pkl(lyric_clf, 'mnb_notparedData_balancedaccuracy')

    return lyric_clf

def classify_text(model, text):

    print(text)
    print(model.predict_proba(text))
    best_genre = model.predict(text)
    print(best_genre)
    return best_genre

def create_playlist(genre, data, input_text, vocab, num_tracks):
    genre_data = data.loc[data['genre']==genre[0]] #Select genre tracks
    tfidf_matrix = vectorize_it(genre_data, input_text, vocab)
    indices = get_closest_indices(tfidf_matrix,num_tracks)
    track_list = fetch_track_ids(genre_data,indices)
    return track_list

def vectorize_it(genre_data, input_text, vocab):
    input_df = pd.DataFrame(data=[['', input_text, '', '']], columns=['genre', 'lyrics', 'orig_index', 'track_id'])
    genre_data = genre_data.append(input_df, ignore_index=True)

    vector_trans = Pipeline([
        ('ctvect', CountVectorizer(vocabulary=vocab)),
#        ('rrt', RowRemoveTransformer()),
        ('tfidf_trans', TfidfTransformer())
    ])

#    text_vect = ctvect.fit_transform(genre_data.lyrics)

    return vector_trans.fit_transform(genre_data.lyrics)

def get_closest_indices(tfidf_matrix,num_tracks):
    nbrs = NearestNeighbors(n_neighbors=105).fit(tfidf_matrix)
    distances, indices = nbrs.kneighbors(tfidf_matrix[-1,:])
    indices=indices[distances>1] #Remove too-close matches (only 1 word in lyrics)
    indices = indices.flatten()[:num_tracks]
    return indices

def fetch_track_ids(genre_data,indices):
    similar_tracks = pd.Series(indices).map(genre_data.reset_index()['track_id'])
    return similar_tracks

def remove_empties(df):
    return df[df['lyrics'] != ' ']

data = load_pkl('labeled_df')
labels = pd.unique(data.genre)
data = remove_empties(data)

#try:
#    genre_clf = load_pkl('cnb_classifier5')
#except FileNotFoundError:
genre_clf = generate_model(data)

input_text = 'Bring up the old country football'

num_tracks=10

total_vocab = genre_clf.named_steps['ctvect'].vocabulary_

best_genre = classify_text(genre_clf, [input_text])

track_list = create_playlist(best_genre, data, input_text, total_vocab, num_tracks)


from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
import db
import datetime
import time
import logging

DRIVER = webdriver.Chrome()
DATE = None
logging.basicConfig(filename="log.log", level=logging.INFO)

ADDRESSES = {
    'soccer': 'https://www.myscore.com.ua/football',
    'hockey': 'https://www.myscore.com.ua/hockey/',
}

def open_selected_sport_completed_games(sport_name):
    c_address = ADDRESSES.get(sport_name)
    logging.info(f"open {c_address}")

    DRIVER.get(c_address)
    ul = DRIVER.find_element_by_css_selector('ul.tabs__list')
    items = ul.find_elements_by_tag_name("li")

    for item in items:
        if item.text == 'Завершенные':
            item.click()
            return

def get_all_leagues(sport_name):
    games = {}
    try:
        games_list = DRIVER.find_element_by_css_selector(f'div.sportName.{sport_name}')
        logging.info(f"there are completed {sport_name} games for today")

    except NoSuchElementException:
        logging.info(f"there are no completed {sport_name} games today")
        return

    items = games_list.find_elements_by_css_selector('div.event__header')
    (games_list.find_elements_by_class_name('event__match.event__match--oneLine'))

    for extended_item in items:
        try:
            extended_item.find_element_by_css_selector('div.event__expander.icon--expander.collapse').click()
        except NoSuchElementException:
            pass

    for item in items:
        title_box = item.find_element_by_css_selector('div.event__titleBox')
        country = title_box.find_element_by_css_selector('span.event__title--type').text
        league_name = title_box.find_element_by_css_selector('span.event__title--name').text
        item.find_element_by_css_selector('div.event__expander.icon--expander.expand').click()
        current_league_games = games_list.find_elements_by_css_selector('.event__match.event__match--oneLine')
        for game in current_league_games:
            id_ = game.get_attribute('id')
            stage = game.find_element_by_css_selector('div.event__stage').text
            if stage != 'Завершен':
                # delete canceled games
                current_league_games.remove(game)
                continue
            participant_home = game.find_element_by_css_selector('div.event__participant.event__participant--home').text
            participant_away = game.find_element_by_css_selector('div.event__participant.event__participant--away').text

            score_1, score_2 = None, None
            if sport_name == 'soccer':
                score_1, score_2 = game.find_elements_by_css_selector('span')
            elif sport_name == 'hockey':
                score_1 = game.find_element_by_css_selector('div.event__score.event__score--home').text
                score_2 = game.find_element_by_css_selector('div.event__score.event__score--away').text

            if score_1 is None or score_2 is None:
                continue

            game_obj = db.Game(id_, sport_name, participant_home, participant_away, score_1.text, score_2.text, country,
                               league_name)
            games[id_] = game_obj

        if len(current_league_games) == 0:
            # Deleting a record of the league if there were only canceled games in it
            items.remove(item)

        item.find_element_by_css_selector('div.event__expander.icon--expander.collapse').click()
    return games

if __name__ == '__main__':

    logging.info("---------Parser started---------")
    db.create_table()
    DATE = datetime.date.today()
    logging.info(f"set DATE {DATE}")

    while True:
        if DATE != datetime.date.today():
            db.drop_table()
            db.create_table()
            DATE = datetime.date.today()
            logging.info(f"set DATE {DATE}")

        for game_name in ADDRESSES:
            logging.info(f"started get info about {game_name}")
            open_selected_sport_completed_games(game_name)
            games = get_all_leagues(game_name)
            db.add_games(games)

        time.sleep(3600)


#!/usr/bin/python3
"""
This Handles all the api Restful actions for Customers
"""
from flask import jsonify, request, make_response
from api.v1.views import app_views
from models.customer import Customer
from models import storage
from flask_jwt_extended import jwt_required, get_jwt_identity, create_access_token
from datetime import timedelta
from api.v1.errors import error_response, bad_request, not_found

@app_views.route('/customer_login', methods=['POST'], strict_slashes=False)
def customer_login():
    """ This returns a token for an authenticated customer """

    form_request = request.form

    if not form_request:
        return bad_request("Not a JSON")
    if 'phone_no' not in form_request:
        return bad_request("Missing Phone Number")

    for customer in storage.all(Customer).values():
        if customer.phone_no == form_request['phone_no']:
            expires = timedelta(seconds=3600)
            access_token = create_access_token(identity=customer.phone_no, expires_delta=expires)
            return make_response(jsonify({'access_token': access_token, 'id': customer.id}), 200)

    return bad_request("Invalid credentials")

@app_views.route('/customers', methods=['GET'], strict_slashes=False)
@jwt_required()
def get_customers():

    """This retrieves a list all customers"""
    customers_list = [cust.to_dict() for cust in storage.all(Customer).values()]
    return jsonify(customers_list)

@app_views.route('/customers/<customer_id>', methods=['GET'], strict_slashes=False)
def get_customer(customer_id):
    """This retrieves a customer based on its id"""

    customer = storage.get(Customer, customer_id)
    if not customer:
        return not_found("customer not found")
    return jsonify(customer.to_dict())

@app_views.route('/customers', methods=['POST'], strict_slashes=False)
def post_customer():
    """This creates a new customer object"""

    form_request = request.form

    if not form_request:
        return bad_request("Not form-data")

    required = ['name', 'phone_no', 'address', 'email']
    for val in required:
        if val not in form_request:
            return bad_request(f"Missing {val}")

    for cust in storage.all(Customer).values():
        if form_request['email'] == cust.email:
            return bad_request("Email Already Exist")
        if form_request['phone_no'] == cust.phone_no:
            return bad_request("Phone Already Exist")

    customer = Customer(**form_request)
    customer.save()
    return make_response(jsonify(customer.to_dict()), 201)

@app_views.route('/customers/<customer_id>', methods=['PUT'], strict_slashes=False)
def put_customer(customer_id):
    """Updates a particular customer's attributes"""

    customer = storage.get(Customer, customer_id)
    if not customer:
        return not_found()
    form_request = request.form
    if not form_request:
        return bad_request("Not form-data")
    ignore_list = ['id', 'updated_at', 'created_at']

    for key, value in form_request.items():
        if key not in ignore_list:
            setattr(customer, key, value)

    storage.save()
    return make_response(jsonify(customer.to_dict()), 200)

@app_views.route('/customers/<customer_id>', methods=['DELETE'], strict_slashes=False)
def delete_customer(customer_id):
    """Deletes a paticular customer object """

    customer = storage.get(Customer, customer_id)
    if not customer:
        return not_found()

    storage.delete(customer)
    storage.save()

    return make_response(jsonify({}), 200)


from django.urls import reverse
from django.test import TestCase
from django.test import Client
from tests.factories.gbe_factories import (
    ProfileFactory,
)
from tests.functions.gbe_functions import (
    grant_privilege,
    is_login_page,
    login_as,
)
from tests.contexts.show_context import ShowContext

class TestTemplateList(TestCase):
    view_name = 'list_template'

    def setUp(self):
        self.client = Client()
        self.privileged_profile = ProfileFactory()
        self.url = reverse(self.view_name,
                           urlconf="gbe.email.urls")

    def test_no_login_gives_error(self):
        response = self.client.get(self.url, follow=True)
        redirect_url = "%s?next=%s" % (reverse('login'), self.url)
        self.assertRedirects(response, redirect_url)
        self.assertTrue(is_login_page(response))

    def test_no_priv(self):
        login_as(ProfileFactory(), self)
        response = self.client.get(self.url)
        self.assertEqual(response.status_code, 403)

    def test_volunteer_coordinator(self):
        grant_privilege(self.privileged_profile.user_object,
                        'Volunteer Coordinator')
        login_as(self.privileged_profile, self)
        response = self.client.get(self.url)

        templates = ["volunteer accepted",
                     "volunteer awaiting approval",
                     "volunteer duplicate",
                     "volunteer no decision",
                     "volunteer reject",
                     "volunteer schedule update",
                     "volunteer changed schedule",
                     "volunteer submission notification",
                     "volunteer update notification",
                     "volunteer wait list",
                     "volunteer withdrawn", ]
        for template in templates:
            self.assertContains(response, template)

    def test_act_coordinator(self):
        context = ShowContext()
        grant_privilege(self.privileged_profile.user_object,
                        'Act Coordinator')
        login_as(self.privileged_profile, self)
        response = self.client.get(self.url)

        templates = ["act accepted - %s" % context.sched_event.title.lower(),
                     "act duplicate",
                     "act no decision",
                     "act reject",
                     "act submission notification",
                     "act wait list - %s" % context.sched_event.title.lower(),
                     "act withdrawn", ]
        for template in templates:
            self.assertContains(response, template)

    def test_basic_bid_coordinators(self):
        for bid_type in ['Class', 'Costume', 'Vendor']:
            grant_privilege(self.privileged_profile.user_object,
                            '%s Coordinator' % bid_type)
            login_as(self.privileged_profile, self)
            response = self.client.get(self.url)

            templates = ["%s accepted",
                         "%s duplicate",
                         "%s no decision",
                         "%s reject",
                         "%s submission notification",
                         "%s wait list",
                         "%s withdrawn", ]
            for template in templates:
                self.assertContains(response, template % bid_type.lower())

    def test_scheduler(self):
        grant_privilege(self.privileged_profile.user_object,
                        'Scheduling Mavens')
        login_as(self.privileged_profile, self)
        response = self.client.get(self.url)
        self.assertContains(response, "daily schedule")

    def test_registrar(self):
        grant_privilege(self.privileged_profile.user_object, 'Registrar')
        login_as(self.privileged_profile, self)
        response = self.client.get(self.url)
        self.assertContains(response, "unsubscribe email")


import numpy as np
from scipy.optimize import curve_fit
from astropy.io import fits
import matplotlib.pyplot as plt
import time
import multiprocessing

t1 = time.time()

def v1(x):
    return x

def v2(x):
    return ((-3/8)*x+x**2/2)

def objective(t,p,q):
#    print(t)
    V1 = v1(t)
    V2 = v2(t)

    return -p*V1+q*V2

def Fitting(channel):
    pvs = []
    qvs = []
    er_p = []
    er_q = []
    counts = []

    dc = np.load('/home/varghese/Desktop/DS5_DP1/Dataset/Avg_0G/AvgDataCube_0G_{}.npy'.format(channel))

    Sigma2 = np.load('/home/varghese/Desktop/DP1/Codes_Dataset_3/9_readout_error/Long_exp_Variance_channel_{}.npy'.format(channel))
    Sigma1 = np.median(np.sqrt(Sigma2.flatten()))+np.zeros(20)
    fiber_mask = np.load('/home/varghese/Desktop/DP1/Masks/Needed_fiber_mask.npy')[:,512*channel:512*(channel+1)]

    interval = 100
    for flux_bin in range(500,60000,interval):

        pix_mask = (dc[-1] > (flux_bin-interval//2)) & (dc[-1] < (flux_bin+interval//2)) & (fiber_mask == 1)

        avg_count = np.nanmedian(dc[10,pix_mask])

        delta_change_cube = np.nanmedian((dc[10:,pix_mask]-dc[10,pix_mask])*100/dc[10,pix_mask],axis=1)
        time1 = np.arange(0,np.size(delta_change_cube,axis=0),1) / (np.size(delta_change_cube,axis=0))

        if not np.isnan(avg_count) and delta_change_cube[-1]<0:
            Sigma = Sigma1*100/(avg_count*np.sqrt(np.sum(pix_mask)))

            try:
                fitting_data_variables, error = curve_fit(objective,time1,delta_change_cube,np.array([1.0,0.5]),sigma = Sigma)

                p,q = fitting_data_variables

                dp = np.sqrt(error[0][0])
                dq = np.sqrt(error[1][1])

                plt.figure(figsize=(16,9))
                plt.plot(time1,delta_change_cube,'o-',color='blue')
                plt.plot(time1,objective(time1,p,q),'--',color='green')
                plt.xlabel('time')
                plt.ylabel('count')
                plt.title('Count level:{}'.format(flux_bin))
                plt.savefig('Ch_{0}/Fitted_{1}.png'.format(channel,flux_bin))

                pvs.append(p)
                qvs.append(q)
                counts.append(avg_count)
                er_p.append(dp)
                er_q.append(dq)
            except Exception as e:
                print(e)

                print('{0},{1}'.format(avg_count,channel),'cannot be done')

    t2 = time.time()

    print("channel", channel, 'done with initials',(t2-t1)//60, 'min', int(t2-t1)%60,'s')

    np.save('pvs/pvs_{}.npy'.format(channel),pvs)
    np.save('qvs/qvs_{}.npy'.format(channel),qvs)
    np.save('counts/counts_{}.npy'.format(channel),counts)
    np.save('er_p/er_p_{}.npy'.format(channel),er_p)
    np.save('er_q/er_q_{}.npy'.format(channel),er_q)

process1 = multiprocessing.Process(target=Fitting,args = [0])
process2 = multiprocessing.Process(target=Fitting,args = [1])
process3 = multiprocessing.Process(target=Fitting,args = [2])
process4 = multiprocessing.Process(target=Fitting,args = [3])

process1.start()
process2.start()
process3.start()
process4.start()

process1.join()
process2.join()
process3.join()
process4.join()

#for channel in range(0,4,1):
#    Fitting(channel)


"""Fetch paralogs from Ensembl

Example usage:
python3 src/get_paralogs.py --reuse
"""

import argparse
import os
import re
from time import sleep
import json as ljson
import gzip
import csv
import traceback

import requests

from lib import repo, module, ctx, organisms

# # Enable importing local modules when directly calling as script
# if __name__ == "__main__":
#     cur_dir = os.path.join(os.path.dirname(__file__))
#     sys.path.append(cur_dir + "/..")

# from lib import download_gzip

def fetch_genes(organism):
    """List genes symbols for an organism
    """

    ids_by_gene = {}
    genes_by_id = {}
    genes = []
    prefix = ''
    # E.g. https://raw.githubusercontent.com/eweitz/ideogram/master/dist/data/cache/homo-sapiens-genes.tsv
    genes_url = (
        "https://raw.githubusercontent.com/eweitz/ideogram/"
        f"master/dist/data/cache/genes/{slug(organism)}-genes.tsv.gz"
    )
    print('genes_url')
    print(genes_url)
    response = requests.get(genes_url)

    if response.status_code != 200:
        print(f"Status code {response.status_code} for {genes_url}")
        return [genes, prefix, ids_by_gene, genes_by_id]

    tsv_string = gzip.decompress(response.content).decode('utf-8')
    # print('tsv_string')
    # print(tsv_string)
    reader = csv.reader(tsv_string.splitlines(), delimiter="\t")
    for row in reader:
        if row[0][0] == '#':
            if '## prefix' in row[0]:
                prefix = row[0].split('prefix: ')[1]
                continue
        if len(row) < 4: continue
        # row: [chr, start, length, slim_id, symbol, description]
        gene = row[4] # more formally, gene symbol
        id = row[3]
        ids_by_gene[gene] = id
        genes_by_id[id] = gene
        genes.append(gene)

    return [genes, prefix, ids_by_gene, genes_by_id]

def slug(value):
    return value.lower().replace(" ", "-")

def lossy_optimize_paralogs(json_str):
    try:
        json = ljson.loads(json_str)
    except Exception as e:
        return []

    if "error" in json:
        return []

    trimmed_ids = []
    for h in json["data"][0]["homologies"]:
        trimmed_id = re.sub(r'[A-Za-z]+0+', '', h["id"])
        trimmed_ids.append(int(trimmed_id))
    return trimmed_ids

class EnsemblCache():

    def __init__(self, output_dir="data/", reuse=False):
        self.output_dir = output_dir
        self.tmp_dir = f"tmp/"
        self.reuse = reuse

        if not os.path.exists(self.output_dir):
            os.makedirs(self.output_dir)
        if not os.path.exists(self.tmp_dir):
            os.makedirs(self.tmp_dir)

    def fetch_paralogs(self, organism, genes, gene_dir):
        """For given genes, retrieve paralogs from Ensembl; write each to file
        """

        prev_error_pwids = []
        error_pwids = []

        error_path = gene_dir + "errors.csv"
        if os.path.exists(error_path):
            with open(error_path) as f:
                prev_error_pwids = f.read().split(",")
                error_pwids = prev_error_pwids

        for gene in genes:
            json_path = gene_dir + gene + ".json"

            if "/" in gene:
                print(f"Skipping gene with slash in name: {gene}")
                error_pwids.append(gene)
                continue

            if self.reuse:
                if os.path.exists(json_path):
                    print(f"Found cache; skip processing {gene}")
                    continue
                elif id in prev_error_pwids:
                    print(f"Found previous error; skip processing {gene}")
                    continue

            url = (
                f"https://rest.ensembl.org"
                f"/homology/symbol/{slug(organism).replace('-', '_')}/{gene}"
                f"?format=condensed&type=paralogues"
                f"&content-type=application/json"
            )

            print('url')
            print(url)

            try:
                sleep(0.1)
                paralogs = requests.get(url).text
            except Exception as e:
                print(f"Encountered error when stringifying JSON for {gene}")
                error_pwids.append(gene)
                with open(error_path, "w") as f:
                    f.write(",".join(error_pwids))
                sleep(0.1)
                continue

            print("Preparing and writing " + json_path)

            with open(json_path, "w") as f:
                f.write(paralogs)

    def optimize_paralogs(self, genes, ids_by_gene, genes_by_id, tmp_gene_dir, gene_dir):
        optimize_errors = []

        genes_by_paralogs = {}

        num_redundant_paralogs = 0
        seen_genes = {}

        rows = []
        for gene in genes:
        # for gene in genes[:50]: # Helpful for debugging

            # Disregard fusion genes and genes like "AC113554.1"
            if (
                "/" in gene or
                re.match(r'^(AC|AL|BX|AF)[0-9]+\.[0-9]$', gene) #or
                # re.match(r'^RNU[0-9]-', gene)
            ): continue

            # original_name = json_path.split("/")[-1]
            # gene = original_name.split(".json")[0]
            json_path = tmp_gene_dir + gene + '.json'
            # json_path = 'tmp/homo-sapiens/ACE2.json'
            if not os.path.exists(json_path):
                optimize_errors.append(gene)
                continue

            original_gene = gene
            # The same genes are often capitalized differently in different
            # organisms.  We can leverage this to decrease cache size by
            # ~2x.  E.g. human "MTOR" and orthologous mouse "Mtor".
            gene = gene.upper()

            if original_gene in seen_genes:
                continue

            # pwid = re.search(r"WP\d+", name).group() # pathway ID
            optimized_tsv_path = gene_dir + gene + ".tsv"

            # repo_url = f"https://github.com/{repo}/tree/main/"
            # code_url = f"{repo_url}src/{module}"
            # data_url = f"{repo_url}{optimized_tsv_path}"
            # wp_url = f"https://www.wikipathways.org/index.php/Pathway:{pwid}"
            # provenance = "\n".join([
            #     "<!--",
            #     f"  WikiPathways page: {wp_url}",
            #     f"  URL for this compressed file: {data_url}",
            #     # f"  Uncompressed GPML file: {original_name}",
            #     # f"  From upstream ZIP archive: {url}",
            #     f"  Source code for compression: {code_url}",
            #     "-->"
            # ])

            with open(json_path, 'rb') as f:
                json = f.read()

            if json.decode("utf-8") == '{"data": [homologies: []]}':
                print(f"Gene found, but no paralogs for {gene}")
                continue

            # print(f"Optimizing to create: {optimized_tsv_path}")

            try:
                paralogs = lossy_optimize_paralogs(json)

                if len(paralogs) == 0:
                    continue

                if original_gene not in ids_by_gene:
                    print('Missing gene in ids_by_gene: ' + original_gene)
                    # exit()
                    continue

                gene_id = ids_by_gene[original_gene]
                # print('paralogs, genes[:50], genes.index(genes_by_id[str(paralogs[0])])')
                # print(paralogs, genes[:50], genes.index(genes_by_id[str(paralogs[0])]))
                paralogs = list(set(paralogs))
                paralogs.sort()
                paralogs.append(int(gene_id))
                tmp_ids = paralogs
                tmp_ids.sort()
                # tmp_ids = sorted(tmp_ids, key=lambda p: genes.index(genes_by_id[str(p)]))

                tsv = "\t".join([str(i) for i in paralogs])
                tmp_tsv = "\t".join([str(i) for i in tmp_ids])

                # Many genes share all their paralogs (minus themselves) with
                # other genes.  This accounts for that, and compresses the
                # combined paralog cache size by ~3x.
                if "\t" in tsv:
                    if tmp_tsv in genes_by_paralogs:
                        pointer = genes_by_paralogs[tmp_tsv]
                        pointer_id = ids_by_gene[pointer]
                        tsv = '_' + pointer + "\t" + pointer_id
                        num_redundant_paralogs += 1
                    else:
                        genes_by_paralogs[tmp_tsv] = original_gene

                # Omit current gene from its own paralog list
                split_tsv = tsv.split("\t")
                if (gene_id in split_tsv):
                    split_tsv.remove(gene_id)

                tsv = "\t".join(split_tsv)
                tsv = gene_id + "\t" + tsv

                tsv = tsv.encode()
                seen_genes[original_gene] = 1
            except Exception as e:
                handled = "Encountered error converting TSV for gene"
                handled2 = "not well-formed"
                if handled in str(e) or handled2 in str(e):
                    # print('Handled an error')
                    print(e)
                    optimize_errors.append(original_gene)
                    continue
                else:
                    print('Encountered fatal error')
                    print(traceback.format_exc())
                    # raise Exception(e)
                    # print('json')
                    # print(json)
                    # exit()
                    continue

            with open(optimized_tsv_path, "wb") as f:
                f.write(tsv)

            tsv = tsv.decode('utf-8')
            if len(tsv) > 0:
                rows.append(f"{original_gene}\t{tsv}")

        print('num_redundant_paralogs')
        print(num_redundant_paralogs)
        num_errors = len(optimize_errors)
        if num_errors > 0:
            print(f"{num_errors} pathways had optimization errors:")
            print(",".join(optimize_errors))

        return rows

    def populate_by_org(self, organism):
        """Fill caches for a configured organism
        """
        tmp_gene_dir = self.tmp_dir + slug(organism) + "/"
        if not os.path.exists(tmp_gene_dir):
            os.makedirs(tmp_gene_dir)

        gene_dir = self.output_dir  + slug(organism) + "/"
        if not os.path.exists(gene_dir):
            os.makedirs(gene_dir)

        # gpml_dir = self.output_dir +  + organism + /"
        # if not os.path.exists(gpml_dir):
        #     os.makedirs(gpml_dir)

        [genes, prefix, ids_by_gene, genes_by_id] = fetch_genes(organism)
        # self.fetch_paralogs(organism, genes, tmp_gene_dir)
        print('len(genes)')
        print(len(genes))
        rows = self.optimize_paralogs(genes, ids_by_gene, genes_by_id, tmp_gene_dir, gene_dir)

        combined_dir = self.output_dir + "combined/"
        if not os.path.exists(combined_dir):
            os.makedirs(combined_dir)
        combined_path = combined_dir + slug(organism) + "-paralogs.tsv.gz"
        rows.insert(0, f"## prefix: {prefix}")
        combined = "\n".join(rows).encode()
        compressed_combined = gzip.compress(combined)
        with open(combined_path, "wb") as f:
            f.write(compressed_combined)
        print(f"Wrote {combined_path}")

    def populate(self):
        """Fill caches for all configured organisms

        Consider parallelizing this.
        """
        # organisms = ["Homo sapiens", "Mus musculus"] # Comment out to use all
        # organisms = ["Homo sapiens"] # Comment out to use all
        # organisms = ["Mus musculus"] # Comment out to use all
        # organisms = ["Caenorhabditis elegans"] # Comment out to use all
        for organism in organisms:
            self.populate_by_org(organism)

# Command-line handler
if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter
    )
    parser.add_argument(
        "--output-dir",
        help=(
            "Directory to put outcome data.  (default: %(default))"
        ),
        default="data/"
    )
    parser.add_argument(
        "--reuse",
        help=(
            "Whether to use previously-downloaded raw data"
        ),
        action="store_true"
    )
    args = parser.parse_args()
    output_dir = args.output_dir
    reuse = args.reuse

    EnsemblCache(output_dir, reuse).populate()


from django.shortcuts import render, redirect
from django.contrib import messages
import json
from . import stats

regions = ['BR', 'EUNE', 'EUW', 'JP', 'KR', 'LAN', 'LAS', 'NA', 'OCE', 'TR', 'RU']

def main(request):
    return render(request, 'lolstats/main.html', {})

def multisearch(request):
    if request.method == "POST":
        try:
            player_data = stats.multisearch(request)
        except stats.UnableToGetPlayerStats as e:
            messages.error(request, f"ERROR: {e}")
            return redirect('main')
        else:
            return render(request, 'lolstats/multisearched.html', {'player_data':player_data})
    else:
        return render(request, 'lolstats/multisearch.html', {})

def player(request):

    # Get user input and direct to player stats page
    player_name = request.GET.get('player_name')
    region = request.GET.get('region')

    # Ensure region and summoner name are valid
    if region not in regions:
        messages.error(request, "ERROR: Invalid Region")
        return redirect('main')
    if not player_name:
        messages.error(request, "ERROR: Must enter summoner")
        return redirect('main')

    return redirect('player_stats', region, player_name)

def player_stats(request, region: str, player_name: str):
    try:
        player_data = stats.player_data(request, region, player_name)
        player_stats = stats.player_stats(player_data['data'])
    except stats.UnableToGetPlayerStats as e:
        messages.error(request, f"ERROR: {e}")
        return redirect('main')

    # Item data needed for JS
    with open('lolstats/static/json/items_dd_alt.json', 'r', encoding='utf-8') as file:
        item_dd = json.load(file)

    return render(request, 'lolstats/player_stats.html', {'player_stats':player_stats, 'player_data':player_data, 'region':region, 'item_dd':item_dd})

def player_live(request, region: str, player_name: str):
    try:
        player_data = stats.player_live(request, region, player_name)
    except stats.UnableToGetPlayerStats as e:
        messages.error(request, f"ERROR: {e}")
        return redirect('main')
    else:
        return render(request, 'lolstats/player_live.html', {'player_data':player_data, 'region':region, 'player_name':player_name})

def view_404(request, exception=None):
    messages.error(request, "ERROR: 404 Not found")
    return redirect('main')

def view_500(request, exception=None):
    messages.error(request, "ERROR: 500 Server issue")
    return redirect('main')


#!/usr/bin/python3
# -*- coding: utf-8 -*-
# author: JinSong.xiao(61627515@qq.com)
# time: 2019/5/29 18:42

from collections import Counter

def counter_element():
    words = [
        'look', 'into', 'my', 'eyes', 'look', 'into', 'my', 'eyes',
        'the', 'eyes', 'the', 'eyes', 'the', 'eyes', 'not', 'around', 'the',
        'eyes', "don't", 'look', 'around', 'the', 'eyes', 'look', 'into',
        'my', 'eyes', "you're", 'under'
    ]

    word_counts = Counter(words)
    #  Counter 对象可以接受任意的 hashable 序列对象。在底层实现上，一个 Counter 对象就是一个字典，将元素映射到它出现的次数上
    print(word_counts['eyes'], word_counts['look'])
    top_three = word_counts.most_common(3)
    print(top_three)

def counter_calculate(method):
    words = [
        'look', 'into', 'my', 'eyes', 'look', 'into', 'my', 'eyes',
        'the', 'eyes', 'the', 'eyes', 'the', 'eyes', 'not', 'around', 'the',
        'eyes', "don't", 'look', 'around', 'the', 'eyes', 'look', 'into',
        'my', 'eyes', "you're", 'under'
    ]
    morewords = ['why', 'are', 'you', 'not', 'looking', 'in', 'my', 'eyes']
    word_counts = Counter(words)
    # 1
    if method == 1:
        for word in morewords:
            word_counts[word] += 1
        print(word_counts['eyes'], word_counts['look'])
    # 2
    elif method == 2:
        word_counts.update(morewords)
        print(word_counts['eyes'], word_counts['look'])
    #
    else:
        a = Counter(words)
        b = Counter(morewords)
        c = a + b
        d = a - b
        print(c, d)

if __name__ == '__main__':
    counter_element()
    counter_calculate(1)
    counter_calculate(2)
    counter_calculate(3)


from aiogram import Bot, Dispatcher, executor, types
from aiogram.types import InlineQueryResultArticle, InputTextMessageContent
from config import TOKEN_API
import hashlib

bot = Bot(TOKEN_API)
dp = Dispatcher(bot)

@dp.inline_handler()
async def inline_cmd(inline_query: types.InlineQuery):
    text = inline_query.query or 'Empty'
    result_bold = hashlib.md5(text.encode()).hexdigest()
    text_italic = text + '1'
    result_italic = hashlib.md5(text_italic.encode()).hexdigest()

    item_bold = InlineQueryResultArticle(
        id=result_bold,
        input_message_content= InputTextMessageContent(f'{text}',parse_mode='HTML'),
        title = 'Bold',
        description='Empty',
        thumb_height=150,
        thumb_width=150,
        thumb_url = 'https://illustratorhow.com/wp-content/uploads/bold-text-illustrator7.png',
    )

    item_italic = InlineQueryResultArticle(
        id=result_italic,
        input_message_content=InputTextMessageContent(f'{text}',
                                                      parse_mode='HTML'),
        title = 'Italic',
        description='Empty',
        thumb_height=150,
        thumb_width=150,
        thumb_url = 'https://cdn-icons-png.flaticon.com/512/59/59377.png',
    )

    await bot.answer_inline_query(inline_query_id=inline_query.id,
                                  results=[item_bold, item_italic],
                                  cache_time=1)

if __name__ == "__main__":
    executor.start_polling(dispatcher=dp,
                           skip_updates=True)


from django import template
from django.conf import settings
from django.templatetags.static import static
from Preprocesser import Preprocesser
from SearchEngine import SearchEngine
from DocEmbedder import DocEmbedder
from CollectionExplorer.models import Collection, Document

register = template.Library()

static_path = settings.BASE_DIR + "/CollectionExplorer" + static("CollectionExplorer/")
index_path = static_path + "/index/"
model_path = static_path + "/models/"

@register.simple_tag
def ft_search(collection, query):
    se = SearchEngine(index_path=index_path, collection_id=collection.id)
    search_results = se.search(query, limit=100)

    res = search_results["result"]
    searcher = search_results["searcher"]

    results = []

    for hit in res[0:res.scored_length()]: #
        try:
            d = Document.objects.get(path=hit["path"])
        except Document.MultipleObjectsReturned:
            d = Document.objects.filter(path=hit["path"]).order_by('id').first()

        preview = hit.highlights("content")
        results.append((d, preview))

    searcher.close()

    return results

@register.simple_tag
def get_similar_words(query, collection_id):
    model_file = model_path + str(collection_id) + ".model"
    q = {1: query}

    pp = Preprocesser()
    pp.tokenize(q.items())
    query_tokenized = pp.corpus_tokenized

    embedder = DocEmbedder()
    model = embedder.run(path=model_file)

    words = {word: model.wv.most_similar(word, topn=10) for v_list in query_tokenized.values() for word in v_list if word in model.wv.vocab}
    return words.items()