import requests
import discord
from webdriver import keep_alive
from bs4 import BeautifulSoup
import pandas as pd
from discord.ext import commands

bot = commands.Bot(command_prefix='!')
bot.remove_command("help")

@bot.event
async def on_ready():
	await bot.change_presence(status=discord.Status.online, activity=discord.Activity(type=discord.ActivityType.watching, name=""))
	print(f'Logged in as {bot.user.name}')

@commands.command(name="walmart")
async def walmart(ctx, SKU, ZIP):
    url = 'https://brickseek.com/walmart-inventory-checker/'
    payload = {'search_method': 'sku', 'sku': SKU, 'zip': ZIP, 'sort': 'distance'}
    df_record = pd.DataFrame(columns=['Store','City','Availability','Quantity'])
    r = requests.post(url, data=payload).text    # Make a POST request with data
    bs = BeautifulSoup(r, 'html.parser')
    j=0
    a = bs.find_all('div', class_='table__body')
    if a == []:
        print('No results found in the searched area.')
    else:
        store=[]
        city=[]
        q=[]
        stock=[]
        for tag in bs.find_all('div', class_='table__body'):
            for i in range(20):
                m_Store = tag.findAll('strong', class_='address-location-name')
                m=str(m_Store)
                if  i < m.count('/strong'):
                    m_s= m_Store[i].get_text().replace("\nWalmart","")
                    m_add = tag.findAll('address',class_="address")
                    m_Address = m_add[i].contents[2]
                    m_Availability = tag.findAll('span',class_="availability-status-indicator__text")
                    m_a = m_Availability[i].get_text()
                    if m_a =='Out of Stock'or m_a == 'Limited Stock':
                        m_q = str(0)
                        j=j-1
                    else:
                        m_Quantity = tag.findAll('span',class_="table__cell-quantity")
                        m_q = m_Quantity[j].get_text()[9:]
                    j=j+1
                    df_record = df_record.append({'Store':m_s, 'City':m_Address, 'Availability':m_a, 'Quantity':m_q }, ignore_index=True)
                    store.append(str(m_s))
                    city.append(str(m_Address))
                    q.append(str(m_a))
                    stock.append(str(m_q))
                else:
                    # df_record=str(df_record)
                    # df_record=str(df_record)
                    # df_record=str(df_record)
                    # df_record=str(df_record)
                    # df_record=str(df_record)
                    # df_record=str(df_record)
                    # df_record=str(df_record)
                    # df_record=str(df_record)
                    print(store)
                    print(city)
                    print(q)
                    print(str(df_record))
                    print()
                    break
        s='\n'
        store = s.join(store)
        city = s.join(city)
        q = s.join(q)
        stock = s.join(stock)
        embed1 = discord.Embed(title='Walmart Stock Checker', color=3447003)
        embed1.add_field(name = 'Store', value=store , inline = True)
        embed1.add_field(name = 'City', value=city , inline = True)
        embed1.add_field(name = 'Availability', value=q , inline = True)
        # embed1.add_field(name = 'Quantity', value=stock , inline = True)
        r = requests.get("https://brickseek.com/walmart-inventory-checker/?sku={}".format(SKU))
        soup = BeautifulSoup(r.content, 'html.parser')
        for tag in soup.find_all("div", "item-overview__image-wrap"):
        	link = tag.img.get("src")
        s=str(link)
        embed1.set_thumbnail(url=s)
        embed1.set_footer(text='odin#9999')
        await ctx.channel.send(embed=embed1)

@commands.command(name="target")
async def target(ctx, SKU, ZIP):
    SKU=str(SKU)
    if '-' not in SKU:
        SKU = ('{}-{}-{}'.format(SKU[0:3], SKU[3:5], SKU[5:9]))
    url = 'https://brickseek.com/target-inventory-checker/'
    payload = {'search_method': 'sku', 'sku': SKU, 'zip': ZIP, 'sort': 'distance'}
    r = requests.post(url, data=payload).text
    bs = BeautifulSoup(r, 'html.parser')
    print(" Store                Availability      Quantity ")
    j=0
    list1=[]
    for tag in bs.find_all('div', class_='table__body'):
        for i in range(10):
            #print(tag)
            m_Store = tag.findAll('strong', class_='address-location-name')
            m=str(m_Store)
            if i < m.count('/strong'):
                m_s= m_Store[i].get_text()
                m_add = tag.findAll('address',class_="address")
                m_Address = m_add[i].contents[0]
                m_Availability = tag.findAll('span',class_="availability-status-indicator__text")
                m_a = m_Availability[i].get_text()
                m_q = 'Unknown'
                j=j+1
                embeder = m_s+"   "  +  m_Address   + "      " + m_a + "        " + m_q
                list1.append(embeder)
            else:break
    s='\n'
    targ = s.join(list1)
    r = requests.get("https://brickseek.com/target-inventory-checker/?sku={}".format(SKU))
    soup = BeautifulSoup(r.content, 'html.parser')
    for tag in soup.find_all("div", "item-overview__image-wrap"):
    	link = tag.img.get("src")
    link=str(link)
    embed1 = discord.Embed(title='Target Stock Checker',description=targ,color=3447003)
    embed1.set_thumbnail(url=link)
    embed1.set_footer(text='odin#9999')
    # embed1.set_image("https://images-ext-2.discordapp.net/external/tKWiJKemxuuUUMoiRarsbbJGCABzsHXGHGFnkzBF5_g/%3Fwidth%3D608%26height%3D612/https/media.discordapp.net/attachments/765387136122880021/783219912453128202/13753__3_.png")
    await ctx.channel.send(embed=embed1)

@commands.command(name="checking")
async def checking(ctx):
	embed = discord.Embed(title='Stock Checker Instructions', description='**Requesting Channel**: <#783828598129295452>\n\n**__Walmart Stock Checker__**\n\nUsage:\n```!walmart sku ZIP```\n\nExample: ```!walmart 781200042 95928 ```\n\n**__Target Stock Checker__**\n\nUsage\n```!target DPCI Zip```\n\nExample: ```!target 057-10-0162 95928```', color=0x32a852)
	embed.set_thumbnail(url="https://media.discordapp.net/attachments/765387136122880021/783219912453128202/13753__3_.png?width=608&height=612")
	await ctx.send(embed=embed)

bot.add_command(walmart)
bot.add_command(target)
bot.add_command(checking)
keep_alive()

bot.run('token')


"""mysite URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/3.2/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
# -*- coding: utf-8 -*-
from django.urls import path
from . import views
from main.api import api_views
from django.conf import settings
from django.conf.urls.static import static
from rest_framework_simplejwt.views import (
    TokenObtainPairView,
    TokenRefreshView,
)

urlpatterns = [
    path('', views.MainMenuView.as_view(), name="index"),
    path('main_menu/', views.MainMenuView.as_view(), name="main_menu"),

    path('login/', views.CustomLoginView.as_view(), name='login'),
    path('logout/', views.CustomLogoutView.as_view(), name='logout'),
    path('registration/', views.CustomRegistrationView.as_view(), name='registration'),

    # objects urls
    path('objects/add', views.ObjectCreateView.as_view(), name='objects_add'),
    path('objects/<int:pk>', views.ObjectDetailView.as_view(), name='objects_detail'),
    path('objects/edit/<int:pk>', views.ObjectEditView.as_view(), name='objects_edit'),

    # tasks urls
    path('tasks/add', views.TaskCreateView.as_view(), name='tasks_add'),
    path('tasks/<int:pk>', views.TaskDetailView.as_view(), name='tasks_detail'),
    path('tasks/edit/<int:pk>', views.TaskEditView.as_view(), name='tasks_edit'),

    path('tasks/<int:task_id>/image/delete/<int:pk>', views.TaskImageRemoveView.as_view(), name='tasks_image_delete'),

    # users urls
    path('users/add', views.CustomUserCreateView.as_view(), name='users_add'),
    path('users/<int:pk>', views.CustomUserDetailView.as_view(), name='users_detail'),
    path('users/edit/<int:pk>', views.CustomUserEditView.as_view(), name='users_edit'),
    path('users/edit/password/<int:pk>', views.CustomUserPasswordChangeView.as_view(), name='users_change_password'),

    # watercourses urls
    path('watercourses/add/<int:license_id>', views.WaterCourseCreateView.as_view(), name='watercourses_add'),
    path('watercourses/children/<int:pk>', api_views.WaterCourseChildrenDetailView.as_view(), name='watercourse_children'),
    path('objects/set_watercourses/<int:pk>', views.LicenseWaterCourseCreateView.as_view(), name='license_watercourse_add'),
    path('objects/unset_watercourses/<int:pk>', views.LicenseWaterCourseRemoveListView.as_view(), name='license_watercourse_remove'),
    path('objects/unset_watercourse/<int:license_id>/<int:pk>', views.LicenseWaterCourseRemoveView.as_view(), name='license_watercourse_remove_single'),

    path('watercourses_by_license/<int:license_id>', api_views.WaterCourseListAPIView.as_view(), name='waterourse_by_license'),

    # watercourses urls
    path('lines/add/<int:license_id>', views.LineCreateView.as_view(), name='lines_add'),
    path('objects/set_lines/<int:pk>', views.LineLicenseWaterCourseCreateView.as_view(), name='line_license_watercourse_add'),
    path('objects/unset_lines/<int:pk>', views.LineLicenseWaterCourseRemoveListView.as_view(), name='line_watercourse_remove'),
    path('objects/unset_line/<int:license_id>/<int:pk>', views.LineLicenseWaterCourseRemoveView.as_view(), name='line_watercourse_remove'),

    path('lines/<int:watercourse_id>', api_views.LineListAPIView.as_view(), name='lines_list_by_watercourses'),

    # wells urls
    path('wells/add', views.WellCreateView.as_view(), name='wells_add'),
    path('wells/<int:task_id>/<int:pk>', views.WellDetailView.as_view(), name='wells_detail'),
    path('wells/edit/<int:task_id>/<int:pk>', views.WellEditView.as_view(), name='wells_edit'),
    path('wells/set_welltasks/<int:pk>', views.WellTaskCreateView.as_view(), name='wells_task_add'),

    path('wells_by_line/<int:line_id>', api_views.WellListAPIView.as_view(), name='wells_list_by_line'),

    # layers urls
    path('layers/add', views.LayerCreateView.as_view(), name='layers_add'),
    path('layers/<int:pk>', views.LayerDetailView.as_view(), name='layers_detail'),
    path('layers/edit/<int:pk>', views.LayerUpdateView.as_view(), name='layers_edit'),

    # api urls
    path('api/token/', TokenObtainPairView.as_view(), name='token_obtain_pair'),
    path('api/token/refresh/', TokenRefreshView.as_view(), name='token_refresh'),

    path('api/tasks/', api_views.TaskListView.as_view(), name='task_list'),
    path('api/layer/add', api_views.LayerCreateAPIView.as_view(), name='layer_add'),
    path('api/well/add', api_views.WellCreateAPIView.as_view(), name='well_add'),

    path('api/layer_materials/', api_views.LayerMaterialsListAPIView.as_view(), name='layer_materials_list'),
    path('api/synchronize/', api_views.SyncronizeViewSet.as_view({'post': 'create'}), name='synchronize'),

    # documentation urls
    path('documents/add', views.DocumentationCreateView.as_view(), name='layers_add'),
    path('documents/<int:pk>', views.DocumentationDetailView.as_view(), name='documentation_detail'),
    path('documents/edit/<int:pk>', views.DocumentationUpdateView.as_view(), name='documentation_edit'),

    path('mine/add', views.MineCreateView.as_view(), name='mine_edit'),
    path('mine/<int:pk>', views.MineDetailView.as_view(), name='mineetail'),
    path('mine/edit/<int:pk>', views.MineUpdateView.as_view(), name='mine_edit'),

    path('mine/images/add', api_views.MineImageCreateAPIView.as_view(), name='mine_image_add'),

    path('asd', views.the_view)
]

urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


from abc import ABC, abstractmethod
import json
import os
from datetime import datetime

import requests

from utils import get_USD_conversion_rate, get_from_and_up_salary

class JobSiteAPI(ABC):
    @abstractmethod
    def get_vacancies(self, filter_word):
        pass

class HeadHunterAPI(JobSiteAPI):
    def get_vacancies(self, filter_word):
        params = {
            'text': f'NAME:{filter_word}',
            'per_page': 50,
            'only_with_salary': True,
            'area': 113
        }

        req = requests.get('https://api.hh.ru/vacancies', params)
        jsObj = json.loads(req.content.decode())
        req.close()
        return jsObj

class SuperJobAPI(JobSiteAPI):
    __api_key = os.getenv('SJ_API_KEY')

    def get_vacancies(self, filter_word):
        headers = {
            'X-Api-App-Id': self.__api_key
        }
        params = {
            'keyword': filter_word,
            'no_agreement': 1,
            'country_id': 1,
            'count': 50
        }
        req = requests.get('https://api.superjob.ru/2.0/vacancies/', headers=headers, params=params)
        jsObj = json.loads(req.content.decode())
        req.close()
        return jsObj

class Vacancy:
    def __init__(self, dictionary):
        for key, value in dictionary.items():
            setattr(self, key, value)

    @property
    def published_at(self):
        try:
            date = datetime.strptime(self.published[:10], '%Y-%m-%d')
            return f'{date.day}-{date.month}-{date.year}'
        except ValueError:
            return self.published
        except TypeError:
            date = datetime.fromtimestamp(int(self.published))
            return f'{date.day}-{date.month}-{date.year}'

    @property
    def approximate_salary(self):
        if self.salary_from and self.salary_to:
            mean_salary = (self.salary_from + self.salary_to) // 2
        else:
            mean_salary = max(self.salary_from, self.salary_to)
        if 'usd' == self.currency.lower():
            return mean_salary * get_USD_conversion_rate()
        else:
            return mean_salary

    def __str__(self):
        response_0 = f'Вакансия на должность: {self.name} в компанию {self.employer} в г. {self.area}\n'
        if self.salary_from and self.salary_to:
            response_1 = f'Зарплата от {self.salary_from} до {self.salary_to} {self.currency}\n'
        elif self.salary_from:
            response_1 = f'Зарплата от {self.salary_from} {self.currency}\n'
        else:
            response_1 = f'Зарплата до {self.salary_to} {self.currency}\n'

        response_2 = f'Требования\Описание:\n{self.requirements}\nТип занятости: {self.employment_type}\n' \
                     f'Вакансия опубликована {self.published_at}\n{self.url}\n{"-" * 80}'
        return response_0 + response_1 + response_2

    def __le__(self, other):
        if isinstance(other, Vacancy):
            return self.approximate_salary <= other.approximate_salary
        return None

    def __lt__(self, other):
        if isinstance(other, Vacancy):
            return self.approximate_salary < other.approximate_salary
        return None

    def __ge__(self, other):
        if isinstance(other, Vacancy):
            return self.approximate_salary >= other.approximate_salary
        return None

    def __gt__(self, other):
        if isinstance(other, Vacancy):
            return self.approximate_salary > other.approximate_salary
        return None

class Saver:
    def __init__(self):
        self.vacancies = []

    def add_vacancy(self, vacancy):
        if isinstance(vacancy, Vacancy):
            self.vacancies.append(vacancy)
        else:
            print('Error, next time input valid vacancy')

    def add_hh_vacancies(self, search_query):
        vacancies_hh = HeadHunterAPI().get_vacancies(search_query)
        for vacancy_hh in vacancies_hh['items']:
            try:
                dic = {
                    'name': vacancy_hh['name'],
                    'employer': vacancy_hh['employer']['name'],
                    'url': vacancy_hh['alternate_url'],
                    'area': vacancy_hh['area']['name'],
                    'salary_from': vacancy_hh['salary']['from'] if vacancy_hh['salary']['from'] else 0,
                    'salary_to': vacancy_hh['salary']['to'] if vacancy_hh['salary']['to'] else 0,
                    'currency': vacancy_hh['salary']['currency'],
                    'requirements': vacancy_hh['snippet']['requirement'],
                    'published': vacancy_hh['published_at'],
                    'employment_type': vacancy_hh['employment']['name']
                }
            except KeyError:
                continue
            vacancy = Vacancy(dic)
            self.add_vacancy(vacancy)

    def add_sj_vacancies(self, search_query):
        vacancies_sj = SuperJobAPI().get_vacancies(search_query)
        for vacancy_sj in vacancies_sj['objects']:
            try:
                dic = {
                    'name': vacancy_sj['profession'],
                    'employer': vacancy_sj['client']['title'],
                    'url': vacancy_sj['link'],
                    'area': vacancy_sj['town']['title'],
                    'salary_from': vacancy_sj['payment_from'] if vacancy_sj['payment_from'] else 0,
                    'salary_to': vacancy_sj['payment_to'] if vacancy_sj['payment_to'] else 0,
                    'currency': vacancy_sj['currency'],
                    'requirements': vacancy_sj['candidat'],
                    'published': vacancy_sj['date_published'],
                    'employment_type': vacancy_sj['type_of_work']['title']
                }
            except KeyError:
                continue
            vacancy = Vacancy(dic)
            self.add_vacancy(vacancy)

    def delete_vacancy(self, vacancy):
        try:
            self.vacancies.remove(vacancy)
        except ValueError:
            print('Vacancy does not exist')

    def get_vacancies_by_salary(self, salary: str):
        try:
            salary_ = get_from_and_up_salary(salary)
        except ValueError:
            raise ValueError(
                'Введите запрос в одном из форматов "50 000- 100 000 руб." "1000-2000 USD"  "100_000 руб."')
        filtered_vacancies = []
        if len(salary_) == 2:
            from_, up_to = salary_
            for vacancy in self.vacancies:
                if from_ < vacancy.approximate_salary < up_to:
                    filtered_vacancies.append(vacancy)
        else:
            for vacancy in self.vacancies:
                if vacancy.approximate_salary == salary_[0]:
                    filtered_vacancies.append(vacancy)
        return filtered_vacancies

class JSONSaver(Saver):
    def __init__(self):
        super().__init__()

    def save_to_json(self, file_name):
        string = [elem.__dict__ for elem in self.vacancies]
        with open(file_name, 'w', encoding='utf-8') as file:
            json.dump(string, file, ensure_ascii=False)

    def get_from_json(self, file_name):
        self.vacancies.clear()
        with open(file_name, 'r', encoding='utf-8') as file:
            vacancies_json = json.load(file)
            for dict_json in vacancies_json:
                vacancy = Vacancy(dict_json)
                self.add_vacancy(vacancy)
        return


import csv
import logging
import json
import math
import random
import re
import time
import urllib.request
from pathlib import Path
import sys
from bs4 import BeautifulSoup
import requests

import get_edgar.common.my_csv as mc
import get_edgar.common.utils as utils

logger = logging.getLogger(__name__)

EDGAR_PREFIX = "https://www.sec.gov/Archives/"
SEC_PREFIX = "https://www.sec.gov"

def save_file_info(csv_in,folder):
    """ from csv containing EDGAR records to csv containting EDGAR record and file information
        the new csv file has prefix "info_" to the original csv file name

    Arguments:
        csv_in {[Path]} -- [the csv file that contains EDGAR records]
        folder {[Path]} -- [the folder to save the new csv file]

    Returns:
        {[Path]} -- [the path for the csv created]
    """
    csv_save = folder / f'info_{csv_in.name[6:]}'
    if csv_save.exists() == False:
        logger.info(f"start save file info to {csv_in.name}")
        new_rows = get_file_info(csv_in)
        sorted_rows = mc.multikeysort_int(new_rows,'cik','filing_date')
        mc.save_dict_csv(sorted_rows,csv_save)
        logger.info(f"{csv_save.name} created")
    else:
        logger.info(f"{csv_save} already exists")
    return csv_save

## Add file information and htm link to the index downloaded

def get_file_info(csv_in):
    """ Get form and filer information for all EDGAR records in a csv file

    Arguments:
        csv_in {Path} -- the Path object for the csv file that contains EDGAR records,
                         the columns should include "cik", "filing_date", and "html_index"

    Returns:
        [list] -- [list of dictionaries containing original EDGAR record, form, and filer information]
    """
    logger.debug(f"start add file info to {csv_in.name}")
    new_rows = []
    with open(csv_in, 'r', newline='') as f:
        reader = csv.DictReader(f)
        for row in reader:
            time.sleep(1)
            isoup = get_isoup(row.get('html_index'))
            if isoup == None:
                continue
            links = get_htm_links(isoup, row.get('html_index'))
            for num,link in links:
                row[f'htm_link_{num}'] = link
            form_infos = get_form_info(isoup, row.get('html_index'))
            row.update(form_infos)
            filer_infos = get_filer_info(isoup, row)
            row.update(filer_infos)
            # row['year'] = int(f'{csv_in.name[-8:-4]}')
            new_rows.append(row)
    logger.debug(f"list of records with file info created for {csv_in.name}")
    return new_rows

def get_isoup(page):
    """[Get soup for the EDGAR index page for each EDGAR file]

    Arguments:
        page {str} -- [webpage address]

    Returns:
        [BeautifulSoup] -- [the soup parsed using BeautifulSoup]
    """
    i = 0
    while True:
        try:
            html_index = requests.get(page, headers=utils.headers)
            break
        except (requests.exceptions.HTTPError):
            logger.error(f'try wait a minute to reopen {page}')
            time.sleep(70)
        except Exception:
            logger.error(f"cannot download {page}", exc_info=True)
            i += 1
            if i > 10:
                return None
            time.sleep(70)
    try:
        return BeautifulSoup(html_index.text, 'lxml')
    except Exception:
        logger.error(f'Cannot make soup for {page}', exc_info=True)
        return None

def get_htm_links(soup, index_path):
    """[Get page address for the first webpage from an EDGAR file index page]

    Arguments:
        soup {[BeautifulSoup]} -- [the soup for the EDGAR file index page]
        index_path {[str]} -- [the page address for the EDGAR file index page]

    Returns:
        [str] -- [the page address for the first webpage in the EDGAR file index page]
    """
    try:
        links = soup.find_all(href=re.compile(r"Archives.*\.htm"))
        page_links = enumerate([SEC_PREFIX + link['href'] for link in links],1)
        return [(n,re.sub(r'/ix\?doc=/','/',link)) for (n,link) in page_links]
    except:
        logger.exception(f'Unable to get htm pages for {index_path}')
        return None

form_headers = {
    "type_description":re.compile(r'Form\s(.*)(?=:)', re.IGNORECASE),
    "report_period":re.compile(r'\sPeriod of Report\s(\d{4}-\d{2}-\d{2})',re.IGNORECASE),
    "file_date":re.compile(r'\sFiling Date\s(\d{4}-\d{2}-\d{2})',re.IGNORECASE),
    "accepted_time":re.compile(r'\sAccepted\s(\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2})',re.IGNORECASE),
    "accession_num":re.compile(r'SEC Accession No.\s(\d{10}-\d{2}-\d{6})', re.IGNORECASE),
    "items":re.compile(r'Items\s((Item.*))', re.IGNORECASE),
}

def get_form_info(soup, index_path, headers=form_headers):
    """[Get form information from an EDGAR file index page
        the form information is set through "form_headers"
        ]

    Arguments:
        soup {[BeautifulSoup]} -- [the soup for the EDGAR file index page]
        index_path {[str]} -- [the page address for the EDGAR file index page]

    Returns:
        [dict] -- [dictionary representing the form information on the EDGAR file index page]
    """
    form_infos = {}
    form_texts = []
    form_texts.append(soup.find(id="formName").get_text())
    form_texts.append(soup.find(id="secNum").get_text())
    formgroup = soup.find_all("div", {"class":"formGrouping"})
    for group in formgroup:
        form_texts.append(group.get_text())
    tems = headers.items()
    for form_text in form_texts:
        for k, v in tems:
            matches = v.search(form_text)
            if matches:
                form_infos[k] = matches.group(1)
    if len(form_infos) == 0:
        logger.warning(f'No form infos for {index_path}')
    elif len(form_infos) < len(headers):
        logger.debug(f'Incomplete form infos for {index_path}')
    return form_infos

filer_headers = {
    "cname":re.compile(r'\n(.+)\(Filer\)', re.IGNORECASE),
    "fcik":re.compile(r'CIK:\s(\d{10})', re.IGNORECASE),
    "sic":re.compile(r'SIC:\s(\d{4})', re.IGNORECASE),
    "irs_num":re.compile(r'IRS No.:\s(\d+)\s', re.IGNORECASE),
    "year_end":re.compile(r'Fiscal Year End:\s(\d{4})', re.IGNORECASE),
    "state_incorp":re.compile(r'State of Incorp.:\s(\w{2})', re.IGNORECASE),
    "type":re.compile(r'Type:\s(.+?)\s(?=\|)', re.IGNORECASE)
}

fcik_pattern = filer_headers.get("fcik")
fcname_pattern = filer_headers.get("cname")

def get_filer_info(soup, record):
    """[Get filer information from an EDGAR file index page
        the filer information is set through "filer_headers"
        the filer cik needs to be the same as cik in the EDGAR file record]

    Arguments:
        soup {[BeautifulSoup]} -- [the soup for the EDGAR file index page]
        record {[dict]} -- [dictionary representing the EDGAR file record]

    Returns:
        [dict] -- [dictionary representing the filer information on the EDGAR file index page]
    """
    filer_infos = {}
    co_filers_cik = []
    co_filers_cname = []
    all_filer_texts = soup.find_all("div",id="filerDiv")
    tems = filer_headers.items()
    for filer_texts in all_filer_texts:
        filer_text = filer_texts.get_text()
        fcik_info = fcik_pattern.search(filer_text)
        if fcik_info:
            fcik = fcik_info.group(1)
            if str(int(fcik)) != record.get("cik"):
                logger.debug(f'found co-filer in {record.get("html_index")}')
                co_filers_cik.append(fcik)
                cname_info = fcname_pattern.search(filer_text)
                if cname_info:
                    cname = cname_info.group(1)
                    co_filers_cname.append(cname.strip())
                continue
        for k, v in tems:
            matches = v.search(filer_text)
            if matches:
                info_raw = matches.group(1)
                info = info_raw.replace('\n', ' ').replace('\r', ' ')
                filer_infos[k] = info.strip()
        filer_infos = {**filer_infos, **extract_address(filer_texts)}
    if len(filer_infos) == 0:
        logger.warning(f'No filer infos for {record.get("html_index")}')
    if co_filers_cik:
        for i in range(len(co_filers_cik)):
            filer_infos[f'co_filers_cik_{i}'] = co_filers_cik[i]
            try:
                filer_infos[f'co_filers_cname_{i}'] = co_filers_cname[i]
            except IndexError:
                filer_infos[f'co_filers_cname_{i}'] = None
    return filer_infos

def select_items(csv_in,filters):
    csv_out = csv_in.resolve().parent / f'item_{csv_in.name[5:]}'
    if csv_out.exists() == False:
        mc.text_filter(csv_in,csv_out,'items',filters)
        logger.info(f'{csv_out} created')
    else:
        logger.info(f'{csv_out} already exists')

def extract_address(filer_info):
    addresses = filer_info.find_all('div','mailer')
    if addresses:
        for address in addresses:
            add_des = address.contents[0].strip()
            mailer_address = []
            phone = None
            adds = address.find_all('span',class_='mailerAddress')
            if adds:
                for add in adds:
                    add_text = add.get_text().strip()
                    if any(c.isalpha() for c in add_text):
                        mailer_address.append(add_text)
                    else:
                        phone = ''.join([c for c in add_text if c.isdigit()])
            if add_des == 'Mailing Address':
                mail_add = ','.join(mailer_address)
            elif add_des == 'Business Address':
                busi_add = ','.join(mailer_address)
                busi_phone = phone
        return {'mail_add':mail_add, 'busi_add':busi_add,'busi_phone':busi_phone}
    return {'mail_add':None, 'busi_add':None,'busi_phone':None}


from django.contrib.auth.decorators import login_required
from django.core.paginator import Paginator
from django.shortcuts import get_object_or_404, redirect, render

from .forms import CommentForm, PostForm
from .models import Follow, Group, Post, User

NUMBER_OF_POSTS: int = 10

def index(request):
    template = 'posts/index.html'
    posts_list = Post.objects.all()
    paginator = Paginator(posts_list, NUMBER_OF_POSTS)
    page_number = request.GET.get('page')
    page_obj = paginator.get_page(page_number)
    context = {
        'page_obj': page_obj,
    }
    return render(request, template, context)

def group_posts(request, slug):
    template = 'posts/group_list.html'
    group = get_object_or_404(Group, slug=slug)
    posts_list = group.posts.all()
    paginator = Paginator(posts_list, NUMBER_OF_POSTS)
    page_number = request.GET.get('page')
    page_obj = paginator.get_page(page_number)
    context = {
        'group': group,
        'page_obj': page_obj
    }
    return render(request, template, context)

def profile(request, username):
    profile_user = get_object_or_404(User, username=username)
    user = request.user
    posts_count = profile_user.posts.count()
    posts_list_username = profile_user.posts.all()
    paginator = Paginator(posts_list_username, NUMBER_OF_POSTS)
    page_number = request.GET.get('page')
    page_obj = paginator.get_page(page_number)
    following = False
    if user.is_authenticated:
        following = None
        if profile_user != user:
            following = False
            if profile_user.following.exists():
                following = True
    context = {
        'user': user,
        'following': following,
        'profile_user': profile_user,
        'page_obj': page_obj,
        'posts_count': posts_count,
    }
    return render(request, 'posts/profile.html', context)

@login_required
def add_comment(request, post_id):
    post = get_object_or_404(Post, id=post_id)
    form = CommentForm(request.POST or None)
    if form.is_valid():
        comment = form.save(commit=False)
        comment.author = request.user
        comment.post = post
        comment.save()
    return redirect('posts:post_detail', post_id)

def post_detail(request, post_id):
    post_user = get_object_or_404(Post, id=post_id)
    posts_count = post_user.author.posts.count()
    comments = post_user.comments.filter(post_id=post_id)
    form_comments = CommentForm(request.POST or None)
    context = {
        'post_user': post_user,
        'posts_count': posts_count,
        'comments': comments,
        'form_comments': form_comments
    }
    return render(request, 'posts/post_detail.html', context)

@login_required
def post_create(request):
    form = PostForm(request.POST or None)
    if form.is_valid():
        post = form.save(commit=False)
        post.author = request.user
        post.save()
        return redirect('posts:profile', request.user)

    return render(request, 'posts/create_post.html', {'form': form})

@login_required
def post_edit(request, post_id):
    post = get_object_or_404(Post, id=post_id)
    if post.author != request.user:
        return redirect('posts:post_detail', post_id=post_id)

    form = PostForm(
        request.POST or None,
        files=request.FILES or None,
        instance=post,
    )
    if form.is_valid():
        post = form.save(commit=False)
        post.author = request.user
        post.save()
        return redirect('posts:post_detail', post.id)

    is_edit = post.text
    context = {
        'is_edit': is_edit,
        'form': form,
        'post': post,
    }
    return render(request, 'posts/create_post.html', context)

@login_required
def follow_index(request):
    author_ids = Follow.objects.filter(user=request.user).values_list(
        'author_id',
        flat=True
    )
    posts_list = Post.objects.filter(author_id__in=author_ids)
    paginator = Paginator(posts_list, NUMBER_OF_POSTS)
    page_number = request.GET.get('page')
    page_obj = paginator.get_page(page_number)
    context = {
        'page_obj': page_obj
    }
    return render(request, 'posts/follow.html', context)

@login_required
def profile_follow(request, username):
    author = get_object_or_404(User, username=username)
    if author != request.user:
        Follow.objects.get_or_create(user=request.user, author=author)
        return redirect('posts:profile', author.username)
    else:
        return redirect('posts:profile', author.username)

@login_required
def profile_unfollow(request, username):
    author = get_object_or_404(User, username=username)
    Follow.objects.filter(user=request.user, author=author).delete()
    return redirect('posts:profile', author.username)


import numpy as np
import torch
import torch.nn as nn

class RNNBaseSTFTMask(nn.Module):
    def __init__(self,
                num_spk=2,
                audio_channels=2,
                n_fft=512,
                hop_length=256,
                sample_rate=16000,
                rnn_hidden=256,
                rnn_layer=2,
                rnn_type="rnn",
                drop_out=0.5,
                activation="relu",
                bidirectional=False,
                *args,
                **kwarg):
        super(RNNBaseSTFTMask, self).__init__()

        self.audio_channels = audio_channels
        self.num_spk = num_spk
        self.n_fft = n_fft
        self.hop_length = hop_length
        self.sample_rate = sample_rate

        n_features = n_fft//2+1
        self.ampltude = Amplitude()
        self.phase = Phase()

        # torch.rnn batch_first
        # https://discuss.pytorch.org/t/could-someone-explain-batch-first-true-in-lstm/15402/9
        # (batch, seq, feature)
        # Without batch_first=True it will use the first dimension as the sequence dimension.
        # With batch_first=True it will use the second dimension as the sequence dimension.
        # out[-1]    # If batch_first=True OR
        # out[:, -1] # If batch_dirst=False
        if rnn_type == "rnn":
            self.rnn = nn.RNN(input_size=n_features,
                        hidden_size=rnn_hidden,
                        num_layers=rnn_layer,
                        bias=False,
                        dropout=drop_out,
                        batch_first=False,
                        bidirectional=bidirectional)
        elif rnn_type == "lstm":
            self.rnn = nn.LSTM(input_size=n_features,
                        hidden_size=rnn_hidden,
                        num_layers=rnn_layer,
                        bias=False,
                        dropout=drop_out,
                        batch_first=False,
                        bidirectional=bidirectional)
        elif rnn_type == "gru":
            self.rnn = nn.GRU(input_size=n_features,
                        hidden_size=rnn_hidden,
                        num_layers=rnn_layer,
                        bias=False,
                        dropout=drop_out,
                        batch_first=False,
                        bidirectional=bidirectional)

        self.batchnorm = nn.BatchNorm1d(num_features=rnn_hidden if not bidirectional else rnn_hidden*2)

        linear = nn.Linear(in_features=rnn_hidden if not bidirectional else rnn_hidden*2,
                                out_features=n_features*num_spk,
                                bias=True)
        if activation=="relu":
            activation = nn.ReLU()

        self.fc_layers = nn.Sequential(
            linear,
            activation,
        )

    def forward(self, inputs):
        # print(inputs.shape)
        mask = self.ampltude(inputs)
        batch, nchannel, nfeature, nframe = mask.shape
        # batch, features, seq
        mask = torch.reshape(mask, shape=(batch*nchannel, nfeature, nframe)) # merge channel
        # print(mask.shape)
        mask = torch.transpose(mask, 1, 2)
        # print(mask.shape)
        mask, _ = self.rnn(mask) # batch, seq, features
        # print(mask.shape)
        mask = torch.transpose(mask, 1, 2)
        # print(mask.shape)
        mask = self.batchnorm(mask) # batch, features, seq
        # print(mask.shape)
        mask = torch.transpose(mask, 1, 2)
        # print(mask.shape)
        mask = self.fc_layers(mask) # batch, seq, features
        # print(mask.shape)
        mask = torch.transpose(mask, -1, -2) # batch, seq, features
        # print(mask.shape)
        batch, nfeature, nframe = mask.shape
        mask = torch.reshape(mask, shape=(batch, self.num_spk, int(nfeature//self.num_spk), nframe))
        # mask = mask.view(batch, self.num_spk, int(nfeature//self.num_spk), nframe)
        # print(mask.shape)
        mask = torch.reshape(mask, shape=(batch//nchannel, nchannel, self.num_spk, int(nfeature//self.num_spk), nframe))
        # mask = mask.view(batch//nchannel, nchannel, self.num_spk, int(nfeature//self.num_spk), nframe)
        # print(mask.shape)
        mask = torch.transpose(mask, 1, 2)
        # print(mask.shape)

        mask = torch.unsqueeze(mask, dim=-1) # dtype expand
        out = mask*torch.unsqueeze(inputs, dim=1)
        return out

class Amplitude(nn.Module):
    def __init__(self,
                *args,
                **kwarg):
        super(Amplitude, self).__init__()
    def forward(self, inputs):
        assert inputs.size()[-1] == 2, f"Tensor needs real and imag in the last rank..."
        return torch.abs(torch.pow(inputs[..., 0], exponent=2) - torch.pow(inputs[..., 1], exponent=2))

class Phase(nn.Module):
    def __init__(
        self,
        *args,
        **kwargs,
    ):
        super().__init__(**kwargs)
        self.eps = torch.tensor(np.ones(1, dtype=np.float32)*1e-5, dtype=torch.float32)

    def call(self, inputs, training=True):
        inputs[..., 1] += self.eps
        outputs = torch.angle(torch.complex(real=inputs[..., 0], imag=inputs[..., 1]))
        return outputs

if __name__ == "__main__":
    # First checking if GPU is available
    train_on_gpu=torch.cuda.is_available()

    def get_model():
        return RNNBaseSTFTMask

    if(train_on_gpu):
        print('Training on GPU.')
    else:
        print('No GPU available, training on CPU.')

    device = torch.device('cuda' if train_on_gpu else 'cpu')

    import argparse
    parser = argparse.ArgumentParser(
        "denoiser.demucs",
        description="Benchmark the streaming Demucs implementation, "
                    "as well as checking the delta with the offline implementation.")
    parser.add_argument("--sample_rate", default=16000, type=int)
    parser.add_argument("--segment", default=1.024, type=float)
    parser.add_argument("--audio_channels", default=2, type=int)
    parser.add_argument("--num_spk", default=2, type=int)
    parser.add_argument("--n_fft", default=512, type=int)
    parser.add_argument("--hop_length", default=256, type=int)

    parser.add_argument("--rnn_hidden", default=896, type=int)
    parser.add_argument("--rnn_layer", default=3, type=int)
    parser.add_argument("--rnn_type", default="lstm", type=str)
    parser.add_argument("--activation", default="relu", type=str)
    parser.add_argument("--bidirectional", default=True, type=bool)
    parser.add_argument("--drop_out", default=0.5, type=float)
    parser.add_argument("--device", default="cpu", type=str)

    args = parser.parse_args()

    model = get_model()(num_spk=args.num_spk,
                    audio_channels=args.audio_channels,
                    n_fft=args.n_fft,
                    hop_length=args.hop_length,
                    sample_rate=args.sample_rate,
                    rnn_hidden=args.rnn_hidden,
                    rnn_layer=args.rnn_layer,
                    rnn_type=args.rnn_type,
                    drop_out=args.drop_out,
                    activation=args.activation,
                    bidirectional=args.bidirectional,
                    ).to(args.device)

    nframe = int(int(args.sample_rate*args.segment) // args.hop_length) + 1
    nfeature = int(args.n_fft//2)+1

    x = torch.randn(args.audio_channels, nfeature, nframe, 2).to(args.device) # channel, T, F, real/imag
    out = model(x[None])[0]
    model_size = sum(p.numel() for p in model.parameters()) * 4 / 2**20
    print(f"model size: {model_size:.1f}MB")


from flask import Flask
from flask import render_template,request
from pymongo import MongoClient
import json
from bson import json_util
from bson.json_util import dumps

app = Flask(__name__,template_folder='/home/sri/Downloads/AAL-94_dataset/AALtorch/template')
print(app)
MONGOD_HOST = 'localhost'
MONGOD_PORT = 27017
DBS_NAME = 'pymongo_test'
COLLECTION_NAME = 'posts'
FIELDS = {'Class': True, 'Conf': True,'Dates': True}
#var date,timestamp

@app.route("/")
def demo1():
    return render_template("demo.html")

@app.route("/pymongo_test/posts")
def donor_projects():
    connection = MongoClient(MONGOD_HOST, MONGOD_PORT)
    collection = connection[DBS_NAME][COLLECTION_NAME]
    projects = collection.find(projection=FIELDS)
    #print(timestamp)
    cursor = []
    cursor = collection.aggregate([{"$group":{"_id":"$Class","Count":{"$sum":1}}}])
    json_projects = []
    count = []
    for document in cursor:
    	count.append(document)
    for project in projects:
        json_projects.append(project)
    json_projects = json.dumps(json_projects, default=json_util.default)
    count = json.dumps(count, default=json_util.default)
    #print(count)
    connection.close()
    return json_projects

if __name__ == '__main__':
    app.run(host='0.0.0.0',port=8000,debug=True)


import os

import pygame

from battle2.model import Direction
from battle2.model import State
import battle2.eventmanager as evm

SCREENWIDTH = 1600
SCREENHEIGHT = 800  # 1600, 800  # 1920, 1080
WINDOWWIDTH = 800
WINDOWHEIGHT = 600
WINDOWPOS = 100, 100

TILESIZE = 32
PLAYERLAYER = 2
GRIDLAYER = 4

FPS = 60

BLACK = pygame.Color("black")
WHITE = pygame.Color("white")
GREEN = pygame.Color("green")
GRAY = pygame.Color("gray38")
DARKGRAY = pygame.Color("gray12")
BLUE = pygame.Color("blue")
PURPLE = pygame.Color("purple")

class GraphicalView(object):
    """
    Draws the model state onto the screen.
    """
    def __init__(self, ev_manager, model):
        """
        ev_manager (EventManager): Allows posting messages to the event queue.
        model (GameEngine): a strong reference to the game Model.

        Attributes:
        isinitialized (bool): pygame is ready to draw.
        screen (pygame.Surface): the screen surface.
        clock (pygame.time.Clock): keeps the fps constant.
        fonts (pygame.Font): a small font.
        """
        self.ev_manager = ev_manager
        ev_manager.register_listener(self)
        self.model = model
        self.isinitialized = False

    def notify(self, event):
        """
        Receive events posted to the message queue.
        """
        if isinstance(event, evm.InitializeEvent):
            self._initialize()

        elif isinstance(event, evm.DrawMapEvent):
            self._draw_map()

        elif isinstance(event, evm.QuitEvent):
            self.isinitialized = False
            pygame.quit()

        elif isinstance(event, evm.CharUpdateEvent):

            self.current_player.rect.topleft = self.model.current_character.new_position
            self.detect_collision()
            self.current_player.updatespeed = self.model.current_character.movespeed
            self.current_player.update(event)

        elif isinstance(event, evm.InputEvent):
            if event.key == pygame.K_c:
                self.model.current_character.align_to_grid()
                # voeg de rect van de huidige speler toe aan de lists en verwijder de rect van start_pos
                self.model.map.add_rect_to_list(self.current_player.rect, self.model.map.heroes)
                self.model.map.add_rect_to_list(self.current_player.rect, self.model.map.obstacles)
                self.model.map.start_pos = None

                self.model.current_character = self.model.characters[1]
                self.current_player = self.players[1]

                # de rect van de player die aan de beurt is weer verwijderen en ken de start_pos toe
                self.model.map.del_rect_from_list(self.current_player.rect, self.model.map.heroes)
                self.model.map.del_rect_from_list(self.current_player.rect, self.model.map.obstacles)
                self.model.map.start_pos = self.current_player.rect.copy()

                # voeg de info sprites toe aan de mapview vanuit de mapdata
                self.map1.info.append(InfoSprite(self.model.map.start_pos, 'start', GRIDLAYER))
                for rect in self.model.map.trees:
                    self.map1.info.append(InfoSprite(pygame.Rect(rect), 'tree', GRIDLAYER))
                for rect in self.model.map.waters:
                    self.map1.info.append(InfoSprite(pygame.Rect(rect), 'water', GRIDLAYER))
                for rect in self.model.map.heroes:
                    self.map1.info.append(InfoSprite(pygame.Rect(rect), 'hero', GRIDLAYER))

                # voeg de obstacle waarden toe aan de mapview vanuit de mapdata
                for rect in self.model.map.obstacles:
                    self.map1.obstacles.append(pygame.Rect(rect))
                for rect in self.model.map.low_obst:
                    self.map1.low_obst.append(pygame.Rect(rect))

            if event.key == pygame.K_F10:
                self.map1.grid.show ^= True
            if event.key == pygame.K_F11:
                self.info ^= True
            if event.key == pygame.K_F12:
                self.debug ^= True          # simple boolean swith

        elif isinstance(event, evm.TickEvent):
            if not self.isinitialized:
                return
            currentstate = self.model.state.peek()
            if currentstate == State.Intro:
                self.render_intro()
            if currentstate == State.Menu:
                self.render_menu()
            if currentstate == State.Play:
                self.render_play()
            if currentstate == State.Help:
                self.render_help()
            self.clock.tick(FPS)            # limit the redraw speed to 60 frames per second

    def _initialize(self):
        """
        Set up the pygame graphical display and loads graphical resources.
        """
        os.environ['SDL_VIDEO_CENTERED'] = '1'
        pygame.init()
        self.screen = pygame.display.set_mode((SCREENWIDTH, SCREENHEIGHT), pygame.DOUBLEBUF)  # | pygame.FULLSCREEN)
        self.background = pygame.Surface(self.screen.get_size())
        self.background.fill(BLACK)
        self.background = self.background.convert()
        self.window = pygame.Surface((WINDOWWIDTH, WINDOWHEIGHT))
        self.window.fill(DARKGRAY)
        self.window = self.window.convert()

        self.clock = pygame.time.Clock()
        self.debugfont = pygame.font.SysFont('courier', 11)
        self.titlefont = pygame.font.SysFont('sans', 25, True)
        self._init_buttons()

        self.players = []
        for character in self.model.characters:
            self.players.append(CharSprite(character.bmp))
        self.current_player = None

        self.info = False
        self.debug = False
        self.isinitialized = True

    def _draw_map(self):
        import pyscroll
        map_layer = pyscroll.BufferedRenderer(self.model.map.map_data, (WINDOWWIDTH, WINDOWHEIGHT), clamp_camera=True)
        self.group = pyscroll.PyscrollGroup(map_layer=map_layer, default_layer=PLAYERLAYER)

        # zet alle players sprites op de juiste posities
        for count, player in enumerate(self.players):
            player.rect.topleft = self.model.characters[count].new_position

        self.current_player = self.players[0]

        self.group.add(self.players)

        # voeg de posities van alles heroes aan de mapdata toe en ook als obstacle
        for char in self.model.characters:
            rect = pygame.Rect(char.new_position[0], char.new_position[1], char.width, char.height)
            self.model.map.add_rect_to_list(rect, self.model.map.heroes)
            self.model.map.add_rect_to_list(rect, self.model.map.obstacles)

        # de rect van de player die aan de beurt is weer verwijderen en ken de start_pos toe
        self.model.map.del_rect_from_list(self.current_player.rect, self.model.map.heroes)
        self.model.map.del_rect_from_list(self.current_player.rect, self.model.map.obstacles)
        self.model.map.start_pos = self.current_player.rect.copy()

        # creeer een mapview
        self.map1 = MapView(self.model.map.width, self.model.map.height)

        # maak van het current hokje een hero-info sprite
        self.map1.current = InfoSprite(self.current_player.rect, 'hero', GRIDLAYER)

        # voeg de info sprites toe aan de mapview vanuit de mapdata
        self.map1.info.append(InfoSprite(self.model.map.start_pos, 'start', GRIDLAYER))
        for rect in self.model.map.trees:
            self.map1.info.append(InfoSprite(pygame.Rect(rect), 'tree', GRIDLAYER))
        for rect in self.model.map.waters:
            self.map1.info.append(InfoSprite(pygame.Rect(rect), 'water', GRIDLAYER))
        for rect in self.model.map.heroes:
            self.map1.info.append(InfoSprite(pygame.Rect(rect), 'hero', GRIDLAYER))

        # voeg de obstacle waarden toe aan de mapview vanuit de mapdata
        for rect in self.model.map.obstacles:
            self.map1.obstacles.append(pygame.Rect(rect))
        for rect in self.model.map.low_obst:
            self.map1.low_obst.append(pygame.Rect(rect))

    def render_intro(self):
        """
        Render the game intro.
        """
        bg_rect = self.background.get_rect()

        somewords = self.titlefont.render('Battle...!', True, GREEN)
        text_rect = somewords.get_rect()
        text_rect.center = bg_rect.width/2, bg_rect.height/2

        self.screen.blit(somewords, text_rect.topleft)
        pygame.display.flip()
        self.screen.blit(self.background, (0, 0))

    def render_play(self):
        """
        Render the game play.
        """
        self.draw_grid()
        self.draw_info()
        self.show_window()
        self.show_debug()
        self.show_buttons()
        pygame.display.flip()
        self.screen.blit(self.background, (0, 0))

    def draw_grid(self):
        if self.map1.grid.show:
            self.group.add(self.map1.grid)
        else:
            self.group.remove(self.map1.grid)

    def draw_info(self):
        if self.info:
            self.map1.current.rect.topleft = self.model.current_character.new_position
            self.group.add(self.map1.current)
            self.group.add(self.map1.info)
        else:
            self.group.remove(self.map1.current)
            self.group.remove(self.map1.info)

    def show_window(self):
        self.group.center(self.current_player.rect.center)
        self.group.draw(self.window)
        self.screen.blit(self.window, WINDOWPOS)

    def show_debug(self):
        if self.debug:
            text = ("FPS:            {}".format(int(self.clock.get_fps())),
                    "step_north:     {}".format(self.model.current_character.step_north),
                    "step_south:     {}".format(self.model.current_character.step_south),
                    "step_west:      {}".format(self.model.current_character.step_west),
                    "step_east:      {}".format(self.model.current_character.step_east),
                    "step_delay:     {}".format(self.model.current_character.step_delay),
                    "last_direction: {}".format(self.model.current_character.last_direction),
                    "move_direction: {}".format(self.model.current_character.move_direction),
                    "movespeed:      {}".format(self.model.current_character.movespeed),
                    "start_pos.x:    {}".format(self.model.map.start_pos[0]
                                                if self.model.map.start_pos is not None else "None"),
                    "old_position.x: {}".format(self.model.current_character.old_position[0]),
                    "new_position.x: {}".format(self.model.current_character.new_position[0]),
                    "start_pos.y     {}".format(self.model.map.start_pos[1]
                                                if self.model.map.start_pos is not None else "None"),
                    "old_position.y  {}".format(self.model.current_character.old_position[1]),
                    "new_position.y  {}".format(self.model.current_character.new_position[1]),
                    "step_count:     {}".format(self.current_player.step_count),
                    "step_animation: {}".format(self.current_player.step_animation),
                    )
            for count, line in enumerate(text):
                self.screen.blit(self.debugfont.render(line, True, WHITE), (0, count * 10))

    def show_buttons(self):
        for button in self.buttons:
            button.draw(self.screen)

    def render_menu(self):
        """
        Render the game menu.
        """
        somewords = self.titlefont.render('You are in the Menu. Space to play. Esc exits.', True, GREEN)
        self.screen.blit(somewords, (0, 0))
        pygame.display.flip()
        self.screen.blit(self.background, (0, 0))

    def render_help(self):
        """
        Render the help screen.
        """
        somewords = self.titlefont.render('Help is here. space, escape or return.', True, GREEN)
        self.screen.blit(somewords, (0, 0))
        pygame.display.flip()
        self.screen.blit(self.background, (0, 0))

    def _init_buttons(self):
        self.button_view = ButtonSprite((SCREENWIDTH-200,   SCREENHEIGHT-300), "V")
        self.button_up = ButtonSprite((SCREENWIDTH-150,     SCREENHEIGHT-300), "Up")
        self.button_down = ButtonSprite((SCREENWIDTH-150,   SCREENHEIGHT-250), "Down")
        self.button_left = ButtonSprite((SCREENWIDTH-200,   SCREENHEIGHT-250), "Left")
        self.button_right = ButtonSprite((SCREENWIDTH-100,  SCREENHEIGHT-250), "Right")
        self.button_cancel = ButtonSprite((SCREENWIDTH-100, SCREENHEIGHT-200), "C")

        self.buttons = [self.button_view, self.button_up, self.button_down, self.button_left, self.button_right,
                        self.button_cancel]

    def detect_collision(self):
        # loop tegen de rand van een obstacle aan
        # er mag maar 1 obstacle in deze lijst zijn
        if len(self.current_player.rect.collidelistall(self.map1.obstacles)) == 1:
            # obj_nr is het nummer van de betreffende obstacle
            obj_nr = self.current_player.rect.collidelist(self.map1.obstacles)
            self.model.current_character.move_side(self.map1.obstacles[obj_nr])
            self.current_player.rect.topleft = self.model.current_character.new_position

        # loop tegen de rand van een low obstacle aan, bijv water
        if len(self.current_player.rect.collidelistall(self.map1.low_obst)) == 1:
            obj_nr = self.current_player.rect.collidelist(self.map1.low_obst)
            self.model.current_character.move_side(self.map1.low_obst[obj_nr])
            self.current_player.rect.topleft = self.model.current_character.new_position

        # loop tegen een obstacle of low_obst aan
        while self.current_player.rect.collidelist(self.map1.obstacles) > -1 or \
                self.current_player.rect.collidelist(self.map1.low_obst) > -1:
            self.model.current_character.move_back()
            self.current_player.rect.topleft = self.model.current_character.new_position

class MapView(object):
    def __init__(self, map_width, map_height):
        self.grid = Grid(map_width, map_height, TILESIZE, GRIDLAYER)
        self.info = []
        self.current = None
        self.obstacles = []
        self.low_obst = []

class CharSprite(pygame.sprite.Sprite):
    # CharSprite extends the pygame.sprite.Sprite class
    def __init__(self, spritesheet):
        pygame.sprite.Sprite.__init__(self)

        self.west_states = {0:  (32, 32, 32, 32), 1: (0, 32, 32, 32), 2: (32, 32, 32, 32), 3: (64, 32, 32, 32)}
        self.east_states = {0:  (32, 64, 32, 32), 1: (0, 64, 32, 32), 2: (32, 64, 32, 32), 3: (64, 64, 32, 32)}
        self.north_states = {0: (32, 96, 32, 32), 1: (0, 96, 32, 32), 2: (32, 96, 32, 32), 3: (64, 96, 32, 32)}
        self.south_states = {0: (32,  0, 32, 32), 1: (0,  0, 32, 32), 2: (32,  0, 32, 32), 3: (64,  0, 32, 32)}

        # Assign the spritesheet to self.full_sprite
        self.full_sprite = pygame.image.load(spritesheet)
        # 'Clip' the sheet so that only one frame is displayed (the first frame of _south_states)
        self.full_sprite.set_clip(pygame.Rect(self.north_states[0]))

        # Create a rect to animate around the screen
        self.image = self.full_sprite.subsurface(self.full_sprite.get_clip())
        self.rect = self.image.get_rect()
        self.mask = pygame.mask.from_surface(self.image)

        self.updatespeed = 0
        self.step_count = 0
        self.step_animation = 0

    def update(self, event):

        if event.movespeed is None:
            if event.last_dir == Direction.North:
                self._clip(self.north_states[0])
            if event.last_dir == Direction.South:
                self._clip(self.south_states[0])
            if event.last_dir == Direction.West:
                self._clip(self.west_states[0])
            if event.last_dir == Direction.East:
                self._clip(self.east_states[0])
        else:
            if event.move_dir == Direction.North:
                self._clip(self.north_states)
            if event.move_dir == Direction.South:
                self._clip(self.south_states)
            if event.move_dir == Direction.West:
                self._clip(self.west_states)
            if event.move_dir == Direction.East:
                self._clip(self.east_states)

        # Update the image for each pass
        self.image = self.full_sprite.subsurface(self.full_sprite.get_clip())

    def _clip(self, clipped_rect):
        if type(clipped_rect) is dict:
            self.full_sprite.set_clip(pygame.Rect(self._get_frame(clipped_rect)))
        else:
            self.step_count = 0
            self.step_animation = 0
            self.full_sprite.set_clip(pygame.Rect(clipped_rect))
        return clipped_rect

    def _get_frame(self, frame_set):
        self.step_count += 1
        if self.step_count % (24 / self.updatespeed) == 1:  # 24 is deelbaar door alle movespeeds
            self.step_animation += 1
            if self.step_animation > 3:
                self.step_animation = 0
        return frame_set[self.step_animation]

class ButtonSprite(pygame.sprite.Sprite):
    def __init__(self, position, caption):
        pygame.sprite.Sprite.__init__(self)

        self.width = 40
        self.height = 40
        self._bgcolor = BLACK
        self._visible = True

        self.image = pygame.Surface((self.width, self.height))
        self.image = self.image.convert()
        self.rect = self.image.get_rect()
        self.rect.topleft = position

        self.font = pygame.font.SysFont('sans', 14)
        self.caption = self.font.render(caption, True, WHITE)
        self.caprect = self.caption.get_rect()
        self.caprect.center = self.rect.width / 2, self.rect.height / 2

        self._update()

    def draw(self, surface):
        if self._visible:
            surface.blit(self.image, self.rect.topleft)

    def _update(self):
        self.image.fill(self.bgcolor)
        pygame.draw.rect(self.image, WHITE, (0, 0, self.width, self.height), 1)
        self.image.blit(self.caption, self.caprect)

    @property
    def bgcolor(self):
        return self._bgcolor

    @bgcolor.setter
    def bgcolor(self, value):
        self._bgcolor = value
        self._update()

    @property
    def visible(self):
        return self._visible

    @visible.setter
    def visible(self, value):
        self._visible = value
        self._update()

class InfoSprite(pygame.sprite.Sprite):
    def __init__(self, rect, rect_type, layer):
        pygame.sprite.Sprite.__init__(self)

        self._layer = layer
        self.image = pygame.Surface((rect.width, rect.height))
        self.image.fill(BLACK)
        self.image.set_colorkey(BLACK)
        self.rect_type = rect_type
        pygame.draw.rect(self.image, self._color, (0, 0, rect.width, rect.height), 1)
        self.image = self.image.convert()
        self.rect = self.image.get_rect()
        self.rect.topleft = rect.topleft

    @property
    def _color(self):
        if self.rect_type == 'start':
            return GREEN
        if self.rect_type == 'hero':
            return BLUE
        if self.rect_type == 'tree':
            return PURPLE
        if self.rect_type == 'water':
            return BLUE

class Grid(pygame.sprite.Sprite):
    def __init__(self, map_width, map_height, tile_size, layer):
        pygame.sprite.Sprite.__init__(self)

        self._layer = layer
        self.image = pygame.Surface((map_width, map_height))
        self.image.fill(BLACK)
        self.image.set_colorkey(BLACK)
        for i in range(0, map_width, tile_size):
            pygame.draw.line(self.image, GRAY, (0, i), (map_width, i))
        for i in range(0, map_height, tile_size):
            pygame.draw.line(self.image, GRAY, (i, 0), (i, map_height))
        self.image = self.image.convert()
        self.rect = self.image.get_rect()

        self.show = False


#%%
import numpy as np
import pandas as pd
from pandas import DataFrame
from retrying import retry

import random
import pickle

class Player(object):

  def __init__(self, name):
    self.name=name

#%%
class CpuPlayer(Player):
  taken_choice=[1,2,3]

  def __init__(self,name='cpu',learning_mode=True, learning_rate=0.1,searching_rate=0.9,estimated_rate=0.8,max_num=50):
    super(CpuPlayer,self).__init__(name)

    self.learning_rate=learning_rate
    self.searching_rate=searching_rate
    self.estimated_rate=estimated_rate
    self.learning_mode=learning_mode
    self.max_num=max_num
    self._bulid_table()

  def _bulid_table(self):
    self.table=DataFrame(
      data=np.zeros((self.max_num,3)),
      columns=CpuPlayer.taken_choice,
      index=range(1,self.max_num+1)
    )

  def get_taken_num(self, current_num):
    row_value=self.table.loc[current_num,:]

    if self.learning_mode and (row_value.all()==0 or np.random.uniform()>self.searching_rate):
      selected=np.random.choice(CpuPlayer.taken_choice)
    else:
      index=row_value.values.argmax()
      selected=CpuPlayer.taken_choice[index]
    return selected

  def learning(self, current,taken_num,state):
    '''
    state: 1表示赢了，0表示输了,None表示未分胜负
    '''

    reward=self._get_reward(state)

    # 找出选择后的3个可能状态
    next_range_value = self._get_next_range_value(current,taken_num, state)

    em_value=reward+self.estimated_rate * next_range_value

    # print(current,taken_num)
    select_value=self.table.loc[current,taken_num]

    # 更新table
    self.table.loc[current,taken_num]+=self.learning_rate*(em_value-select_value)

  def _get_reward(self, state):
    reward=0
    if state==1:
      reward=10 # 因为赢了，给他大奖励
    elif state==0:
      reward=-10 # 因为赢了，给他大惩罚
    return reward

  def _get_next_range_value(self, current,taken_num, state):

    # 如果已经分出胜负，则无需计算状态转移的估计值
    if state is not None:
      return 0

    # 计算当前选择下的可能局面的价值
    end=current-taken_num-1
    start=end-2
    range_df=self.table.loc[start:end,:].copy()

    range_df['all_less']=range_df.apply(lambda x:(x<0).all(),axis=1)
    range_df['all_zero']=range_df.apply(lambda x:(x==0).all(),axis=1)

    # 假如3行中有任意一行全是负数，意味着本次选择有可能会输
    if range_df['all_less'].any():
      return -2

    if range_df['all_zero'].any():
      return 0

    return range_df.iloc[:,:3].max().max()

  def save_model(self,filename='cpu_play.m'):
    with open(filename,'wb') as f:
      pickle.dump(self,f)
      # json.dumps()
      res=f.name

    return res

  @staticmethod
  def load_from_file(filename='cpu_play.m'):
    with open(filename,'rb') as f:
      return pickle.load(f)

#%%
class Referee(object):

  def __init__(self):
    pass

  def ready(self, start_num):
    self.current_num=start_num

  def is_end_game(self):
    return self.current_num<=0

  def get_state(self,taken_num):
    '''
    state: 1表示赢了，0表示输了,None表示未分胜负
    '''
    if self.current_num-taken_num<=0:
      return 0

    if self.current_num-taken_num==1:
      return 1

    return None

  def take_away(self, taken_num):
    self.current_num-=taken_num

class UserPlayer(Player):

  def __init__(self,name='user'):
    super(UserPlayer,self).__init__(name)

  @retry
  def get_taken_num(self, current_num):
    ipt=input(f'当前剩余{current_num}个石子,输入你要拿取的数量(1-3):')

    try:
      num=int(ipt)
    except ValueError as ex :
      print('哥，输入数字呀！')
      raise ex

    if num<1 or num>3:
      print('你要输入1到3的数值')
      raise Exception()
    return num


from django.conf.urls import url

from ..views.oj import (SearchGroupByKeyWordAPI,
                        JoinGroupBySearchAPI,
                        GroupListAndDetailAPI,
                        HomeWorkListAndDetailAPI
                        )

urlpatterns = [
    url(r"^search_group/?$", SearchGroupByKeyWordAPI.as_view(), name="search_group_api"),
    url(r"^join_group/?$", JoinGroupBySearchAPI.as_view(), name="join_group_api"),
    url(r"^groups/?$", GroupListAndDetailAPI.as_view(), name="group_api"),
    url(r"^homework/?$", HomeWorkListAndDetailAPI.as_view(), name="homework_api"),
]


import bibtexparser
import re
import os
from os import path
if path.exists('mytitles.txt'):
    os.remove('mytitles.txt')
with open('MyCollection.bib') as bibtex_file:
    bib_database = bibtexparser.load(bibtex_file)

with open("mytitles.txt","a") as file1:
    for entry in bib_database.entries:
        title = entry['title']
        title = title.replace('{','')
        title = title.replace('}','')
        print(title)
        file1.write('%s\n' % title)


from django.test import TestCase, tag
from django.urls.base import reverse
from edc_model_wrapper import ModelWrapper

from ..models import ActionItem, ActionType
from ..templatetags.action_item_extras import add_action_item_popover
from ..view_mixins import ActionItemViewMixin
from .models import SubjectIdentifierModel

class MyModelWrapper(ModelWrapper):
    next_url_name = 'dashboard_url'

class TestAction(TestCase):

    def setUp(self):
        self.subject_identifier_model = ActionItem.subject_identifier_model
        ActionItem.subject_identifier_model = 'edc_action_item.subjectidentifiermodel'
        self.subject_identifier = '12345'
        SubjectIdentifierModel.objects.create(
            subject_identifier=self.subject_identifier)
        ActionItemViewMixin.action_item_model_wrapper_cls = MyModelWrapper

    def test_view_populates_action_type(self):
        self.assertEqual(ActionType.objects.all().count(), 0)
        ActionItemViewMixin()
        self.assertGreater(ActionType.objects.all().count(), 0)
        ActionItemViewMixin()
        self.assertGreater(ActionType.objects.all().count(), 0)

    def test_view_context(self):
        view = ActionItemViewMixin()
        view.kwargs = dict(subject_identifier=self.subject_identifier)
        context = view.get_context_data()
        self.assertEqual(context.get('open_action_items'), [])

        for action_type in ActionType.objects.all():
            ActionItem.objects.create(
                subject_identifier=self.subject_identifier,
                action_type=action_type)

        view = ActionItemViewMixin()
        view.kwargs = dict(subject_identifier=self.subject_identifier)
        context = view.get_context_data()
        self.assertEqual(len(context.get('open_action_items')),
                         ActionItem.objects.all().count())

    def test_templatetag(self):
        context = add_action_item_popover(
            self.subject_identifier, 'subject_dashboard_url')
        reverse(context.get('action_item_add_url'))


from selenium import webdriver
import pandas as pd
from IPython.display import display
from selenium.webdriver.chrome.options import Options

# inicializa o programa sem aparecer na tela, em segundo plano
chrome_options = Options()
chrome_options.headless = True

# abre o navegador com as opções definidas acima
navegador = webdriver.Chrome(options=chrome_options)

# passo 1: pegar a cotação das moedas
# dolar
navegador.get('https://www.google.com/search?q=cotacao+dolar&oq=cotacao+dolar&aqs=chrome..69i57j35i39j0i512l3j0i433i512j0i512l4.1498j0j9&sourceid=chrome&ie=UTF-8')
cot_dolar = navegador.find_element('xpath', '//*[@id="knowledge-currency__updatable-data-column"]/div[1]/div[2]/span[1]').get_attribute('data-value')

# euro
navegador.get('https://www.google.com/search?q=cotacao+euro&oq=cotacao+euro&aqs=chrome..69i57j0i512l4j0i10i512j0i512l4.1450j0j7&sourceid=chrome&ie=UTF-8')
cot_euro = navegador.find_element('xpath', '//*[@id="knowledge-currency__updatable-data-column"]/div[1]/div[2]/span[1]').get_attribute('data-value')

# ouro
navegador.get('https://www.melhorcambio.com/ouro-hoje#:~:text=O%20valor%20do%20grama%20do,em%20R%24%20314%2C92.')
cot_ouro = navegador.find_element('xpath', '//*[@id="comercial"]').get_attribute('value')
cot_ouro = cot_ouro.replace(',', '.')

# fecha o navegador após pegar a cotação
navegador.quit()

# passo 2: importar e atualizar a base de dados
df = pd.read_excel('Produtos.xlsx')

# atualizar a cotação das moedas no dataframe
df.loc[df['Moeda'] == 'Dólar', 'Cotação'] = float(cot_dolar)    # loc[linha, coluna]
df.loc[df['Moeda'] == 'Euro', 'Cotação'] = float(cot_euro)
df.loc[df['Moeda'] == 'Ouro', 'Cotação'] = float(cot_ouro)

# atualizar o preço de compra e preço de venda
df['Preço de Compra'] = df['Preço Original'] * df['Cotação']
df['Preço de Venda'] = df['Preço de Compra'] * df['Margem']

# formatar os preços
df['Preço de Compra'], df['Preço de Venda'] = df['Preço de Compra'].map('R${:.2f}'.format), df['Preço de Venda'].map('R${:.2f}'.format)
df['Cotação'] = df['Cotação'].map('R${:.2f}'.format)

# passo 3: exportar a base de dados
df.to_excel('Produtos Novo.xlsx', index=False)   # nome do arquivo a ser criado | index=false para não exportar o index junto

df_novo = pd.read_excel('Produtos Novo.xlsx')
display(df_novo)


import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.mlab as mlab
import statistics as st
from math import pow
import math
import os
import numpy as np
from scipy.stats import norm

FILE_PATH=str(os.getcwd()) + '\Data\heart.csv'
df = pd.read_csv(FILE_PATH)

#list of all the column headings
col_heads = df.columns

#dictionary containing column heading as key and mean of all values in that column as value
col_means = dict([(col, df[col].mean()) for col in col_heads])
print("Means of respective columns: ", col_means)

#dictionary containing column heading as key and variance of all values in that column as value
col_variances = dict([(col, pow(df[col].std(), 2)) for col in col_heads])
print("Variances of respective columns: ",col_variances)

#normalisation/ standardisation of all numeric data (using cumulative distribution for z-scores)
normalized_data = []
for col in col_heads:
    row = [((X - col_means[col]) / pow(col_variances[col], 0.5)) for X in df[col].tolist()]
    normalized_data.append(row)

df_normal = pd.DataFrame(normalized_data, columns = list(range(0,1025)))
df_normal = df_normal.transpose()
df_normal.columns = col_heads

#graphical visualisation of normalized data
def plot_normal(df, col):
    feature=[i for i in df[col]]
    feature.sort()

    plt.plot(feature, norm.pdf(feature, 0, 1), color = 'green', label = 'Normal curve')

    # title and axis labels
    plt.xlabel(col)
    plt.ylabel('Frequency')
    plt.title('Normal curve (bell shaped) for feature data')

    plt.show()

#normal plots for normalized data in each column
for columns in col_heads:
    plot_normal(df_normal, columns)

#find the categorical variables from the histograms
categorical_vars = ['sex', 'cp', 'fbs', 'restecg', 'exang', 'slope', 'thal', 'target']

#plot normal curves only for continuous variables
for columns in col_heads:
    if(columns not in categorical_vars):
        plot_normal(df_normal, columns)

#age, trestbps, chol and thalach are approximately normally distributed, trestbps and chol having slightly extended tails


from glob import glob
from langchain.agents import load_tools
from langchain.agents import initialize_agent
from langchain.agents import AgentType
from langchain.llms import OpenAI
import os, sys, openai
from dotenv import load_dotenv
from langchain.tools import tool, Tool
from retrieval import Retrieval
from pyepsilla import cloud

load_dotenv()

retrieval = Retrieval()

client = cloud.Client(
  project_id=os.getenv("PROJECT_ID"),
  api_key=os.getenv("EPSILLA_API_KEY")
)
db = client.vectordb(db_id=os.getenv("DB_ID"))

@tool
def search_api(question: str) -> str:
    """Searches the relevant information from the document set to answer the question."""
    qs = retrieval.rephrase(question=question)
    query_score_dict = {}
    item = retrieval.vector_search(db, question)
    # print(item)
    query_score_dict[question] = item
    for q in qs:
      item = retrieval.vector_search(db, q)
      query_score_dict[q] = item
    # print(query_score_dict)

    ranking_result = retrieval.ranking_fusion(original_query=question, query_score_dict=query_score_dict)
    final_result = retrieval.generate_content_based_on_ranking(ranking_result)
    return final_result

class DocAgent:
    def __init__(self):
        api_key = os.getenv("OPENAI_KEY")
        llm = OpenAI(temperature=0, openai_api_key=api_key)
        tools = [search_api]
        tools = tools + load_tools(["llm-math"], llm=llm)
        self.agent_executor = initialize_agent(
            tools, llm, agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, verbose=True
        )

    def list_docs(self):
        # List PDF files under ./documents/ folder
        ret = []
        files = glob("./documents/*.pdf")
        for pdf in files:
            ret.append(os.path.basename(pdf))
        return ret

    def solve(self, question):
        response = self.agent_executor.invoke(
            {
                "input": question
            }
        )
        return response['output']

    def rephrase(self, question):
        return retrieval.rephrase(question=question)

    def solve_one(self, file, question, questions):
        # Step 1. Search the relevant information from the document to answer the question.
        query_score_dict = {}
        item = retrieval.vector_search_one_doc(db, question, file)
        # print(item)
        query_score_dict[question] = item
        for q in questions:
          item = retrieval.vector_search_one_doc(db, q, file)
          query_score_dict[q] = item
        # print(query_score_dict)

        ranking_result = retrieval.ranking_fusion(original_query=question, query_score_dict=query_score_dict)
        context = retrieval.generate_content_based_on_ranking(ranking_result)

        # Step 2. Use the prompt to answer the question for the document.
        openai.api_key = os.getenv("OPENAI_KEY")
        response = openai.ChatCompletion.create(
          model="gpt-4",
          messages=[
            {
              "role": "system",
              "content": "You are an assistant answering questions for a given document."
            },
            {
              "role": "user",
              "content": f'''
                Answer the Question based on the given Context. Please don't make things up. Ask for more information when needed.

                Context:
                {context}

                Question:
                {question}

                Answer:
                Let's work this out in a step by step way to be sure we have the right answer.
                '''
            }
          ],
          temperature=0,
          max_tokens=256,
          top_p=1,
          frequency_penalty=0,
          presence_penalty=0
        )

        return response['choices'][0]['message']['content']

    def summary(self, question, concated):
        openai.api_key = os.getenv("OPENAI_KEY")
        response = openai.ChatCompletion.create(
          model="gpt-4",
          messages=[
            {
              "role": "system",
              "content": "You are an assistant answering questions for a given document."
            },
            {
              "role": "user",
              "content": f'''
                Answer the Question based on the Analysis Of Each Document. If some documents are not related to the question, please ignore them.

                Analysis Of Each Document:
                {concated}

                Question:
                {question}

                Answer:
                '''
            }
          ],
          temperature=0,
          max_tokens=256,
          top_p=1,
          frequency_penalty=0,
          presence_penalty=0
        )

        return response['choices'][0]['message']['content']

    def can_loop(self, question):
        openai.api_key = os.getenv("OPENAI_KEY")
        response = openai.ChatCompletion.create(
          model="gpt-4",
          messages=[
            {
              "role": "system",
              "content": "You are an assistant answering questions for a large set of documents."
            },
            {
              "role": "user",
              "content": "For the provided question, determine if we can check the documents one by one and make the judgement and answer it purely based on facts from this file, or we need to cross validate with other files. If former, response \"YES\"; if later, response \"NO\"\n\nQuestion: " + question
            }
          ],
          temperature=0,
          max_tokens=256,
          top_p=1,
          frequency_penalty=0,
          presence_penalty=0
        )
        decision = response['choices'][0]['message']['content'] or 'YES'
        return decision == 'YES'

# agent = DocAgent()
# print(agent.list_docs())


import os
from yt_dlp import YoutubeDL
def download_url(path,URL):
    option = {
        "outtmpl":f"{path}"+"%(title)s.%(ext)s"
    }#パスは実行する環境に合わせて
    URLs=[]
    ydl = YoutubeDL(option)
    URLs.append(URL)
    result = ydl.download(URLs)
    return f"{path}"+"%(title)s.%(ext)s"

download_url(os.getcwd(),f"https://www.youtube.com/watch?v=WEYCz553Sto&ab_channel=S2P%5BThaiLyrics%5D%3Ainfinity")


# coding: utf-8
from abc import ABCMeta
import asyncio
import json
from aiohttp import web, MsgType
from bson import json_util
from django.conf import settings
from django.utils.timezone import now
from parkkeeper import models
from parkkeeper.event import async_recv_event, get_sub_socket
from parkkeeper.const import MONIT_SCHEDULE_EVENT, WORK_SCHEDULE_EVENT
from parkworker.const import MONIT_STATUS_EVENT, TASK_EVENT, WORKER_EVENT, WORK_STATUS_EVENT

def start_server():
    app = web.Application()
    add_routes(app)

    loop = asyncio.get_event_loop()
    handler = app.make_handler()
    f = loop.create_server(handler, '0.0.0.0', settings.WEB_SOCKET_SERVER_PORT)
    srv = loop.run_until_complete(f)
    print('serving on', srv.sockets[0].getsockname())
    try:
        loop.run_forever()
    except KeyboardInterrupt:
        pass
    finally:
        loop.run_until_complete(handler.finish_connections(1.0))
        srv.close()
        loop.run_until_complete(srv.wait_closed())
        loop.run_until_complete(app.finish())
    loop.close()

def add_routes(app):
    app.router.add_route('GET', '/monit_schedules', MonitSchedulesHandler().get_handler)
    app.router.add_route('GET', '/work_schedules', WorkSchedulesHandler().get_handler)
    app.router.add_route('GET', '/monits', MonitResultHandler().get_handler)
    app.router.add_route('GET', '/works', MonitResultHandler().get_handler)
    app.router.add_route('GET', '/waiting_tasks', MonitWaitingTaskHandler().get_handler)
    app.router.add_route('GET', '/current_workers', MonitCurrentWorkerHandler().get_handler)

class WebSocketHandler(metaclass=ABCMeta):
    stop_msg = 'close_ws'
    need_background = False
    stop_background_timeout = 1

    async def process_msg(self, msg_text: str):
        print(msg_text)

    async def background(self, ws: web.WebSocketResponse):
        while not ws.closed:
            print(now())
            await asyncio.sleep(1)
        print('Close background')

    async def get_handler(self, request):
        ws = web.WebSocketResponse()
        await ws.prepare(request)

        # start background process if needed
        background_task = None
        if self.need_background:
            loop = asyncio.get_event_loop()
            background_task = loop.create_task(self.background(ws))

        # process ws messages
        while not ws.closed:
            await self._receive_msg(ws)

        # stop background
        if background_task:
            background_task.cancel()

        return ws

    async def _receive_msg(self, ws: web.WebSocketResponse):
        msg = await ws.receive()

        if msg.tp == MsgType.text:
            if msg.data == self.stop_msg:
                print('Got stop msg')
                await ws.close()
            else:
                await self.process_msg(msg.data)
        elif msg.tp == MsgType.close:
            print('websocket connection closed')
        elif msg.tp == MsgType.error:
            print('ws connection closed with exception %s' %
                  ws.exception())

class MonitSchedulesHandler(WebSocketHandler):
    need_background = True
    stop_background_timeout = 0.1

    async def background(self, ws):
        subscriber_socket = get_sub_socket(MONIT_SCHEDULE_EVENT)
        try:
            while True:
                monit_schedule_json = await async_recv_event(subscriber_socket)
                # print('monit_schedule', monit_schedule_json)
                ws.send_str(monit_schedule_json)
        finally:
            subscriber_socket.close()

class WorkSchedulesHandler(WebSocketHandler):
    need_background = True
    stop_background_timeout = 0.1

    async def background(self, ws):
        subscriber_socket = get_sub_socket(WORK_SCHEDULE_EVENT)
        try:
            while True:
                work_schedule_json = await async_recv_event(subscriber_socket)
                # print('work_schedule', work_schedule_json)
                ws.send_str(work_schedule_json)
        finally:
            subscriber_socket.close()

class MonitResultHandler(WebSocketHandler):
    need_background = True
    stop_background_timeout = 0.1

    async def background(self, ws):
        subscriber_socket = get_sub_socket(MONIT_STATUS_EVENT)
        try:
            while True:
                task_json = await async_recv_event(subscriber_socket)
                task_data = json.loads(task_json, object_hook=json_util.object_hook)
                response = _get_task_represent(task_data)
                ws.send_str(json.dumps(response, default=json_util.default))
        finally:
            subscriber_socket.close()

class WorkResultHandler(WebSocketHandler):
    need_background = True
    stop_background_timeout = 0.1

    async def background(self, ws):
        subscriber_socket = get_sub_socket(WORK_STATUS_EVENT)
        try:
            while True:
                task_json = await async_recv_event(subscriber_socket)
                task_data = json.loads(task_json, object_hook=json_util.object_hook)
                response = _get_task_represent(task_data)
                ws.send_str(json.dumps(response, default=json_util.default))
        finally:
            subscriber_socket.close()

class MonitWaitingTaskHandler(WebSocketHandler):
    need_background = True
    stop_background_timeout = 0.1

    async def background(self, ws):
        subscriber_socket = get_sub_socket(TASK_EVENT)
        try:
            while True:
                response = {'waiting_tasks': []}
                waiting_tasks = models.MonitTask.get_waiting_tasks()
                for task in waiting_tasks:
                    task_data = task.get_data()['task']
                    response['waiting_tasks'].append(_get_task_represent(task_data))
                # print('waiting_tasks count', len(response['waiting_tasks']))

                ws.send_str(json.dumps(response, default=json_util.default))

                # waiting new events
                await async_recv_event(subscriber_socket)
        finally:
            subscriber_socket.close()

class MonitCurrentWorkerHandler(WebSocketHandler):
    need_background = True
    stop_background_timeout = 0.1

    async def background(self, ws):
        subscriber_socket = get_sub_socket(WORKER_EVENT)
        try:
            while True:
                response = {'current_workers': []}
                workers = models.CurrentWorker.objects.all()
                for worker in workers:
                    response['current_workers'].append(
                        _get_worker_represent(worker)
                    )
                # print('current_workers count', len(response['current_workers']))

                # print(response)
                ws.send_str(json.dumps(response, default=json_util.default))

                # waiting new events
                await async_recv_event(subscriber_socket)
        finally:
            subscriber_socket.close()

def _get_worker_represent(worker: models.CurrentWorker) -> dict:
    worker_data = {
        'uuid': str(worker.main.uuid),
        'id': worker.main.id,
        'created_dt': worker.main.created_dt.isoformat(sep=' '),
        'host_name': worker.main.host_name,
        'type': worker.main.type,
        'tasks': [],
    }

    for task in worker.get_tasks():
        task_data = _get_task_represent(task.get_data()['task'])
        worker_data['tasks'].append(task_data)

    return worker_data

def _get_task_represent(task: dict) -> dict:
    task_data = {
        'id': task['id'],
        'host_address': task['host_address'],
        'schedule_id': task['schedule_id'],
        'start_dt': None,
        'result_dt': None,
        'extra': None,
        'level': None,
        'worker': None,
    }

    if 'monit_name' in task:
        task_data['monit_name'] = task['monit_name'],

    if 'work_name' in task:
        task_data['work_name'] = task['work_name'],

    if 'start_dt' in task:
        task_data['start_dt'] = task['start_dt'].replace(microsecond=0).isoformat(sep=' ')

    if 'result' in task:
        task_data['result_dt'] = task['result']['dt'].replace(microsecond=0).isoformat(sep=' ')
        task_data['extra'] = task['result']['extra']
        task_data['level'] = task['result']['level']

    if 'worker' in task:
        task_data['worker'] = task['worker']
        task_data['worker']['created_dt'] = task['worker']['created_dt'].replace(microsecond=0).isoformat(sep=' ')

    return task_data


# coding: utf-8

# In[34]:

#Mandelbrot fractal creation program
#e.g. complex fractal shapes with recursive detail at increasing magnifications
#code adapted from example found @ docs.scipy.org/doc/numpy/user/quickstart.html

import matplotlib.pyplot as plt
import numpy as np

def mbrot(h,w,maxit=125): #higher iterations = more complex edges; longer run times

    a,b=np.mgrid[-1.5:1.5:h*1j,-2:2.1:w*1j] #placement of object in plane
    c=a+b*1j
    d=c
    divtime=maxit+np.zeros(d.shape,dtype=float)

    for thing in range(maxit):
        d=d**2+c
        diverge=d*np.conj(d)>2**2
        div_now=diverge & (divtime==maxit)
        divtime[div_now]=thing
        d[diverge]=1

    return divtime

#sizing, visualization
plt.imshow(mbrot(1500,1500),cmap='winter') #cmap options: matplotlib.org/tutorials/colors/colormaps.html
plt.show()


from django.conf.urls import include

from utils.urls import cbv_url_helper as url
from . import views

urlpatterns = [
    url(r'^$', views.SuperAdminHomeView),
    url(r'^stats/$', views.StatsView),
    url(r'^stats/initial/$', views.InitialStatsView),
    url(r'^stats/get/$', views.GetStatsView),

    url(r'^admins/', include('accounts.urls')),
    url(r'^users/', include('users.root_urls')),
    url(r'^clusters/', include('clusters.urls')),

    url(r'^', include('django.conf.urls.i18n')),
]


import logging
from dataclasses import dataclass
from http import HTTPStatus

from paddington import (
    Joint, Track, ErrorEvent, ErrorTypeSwitch, RouteNotFound, SequentialSwitch,
)
from web_framework.app import App, WsgiContext
from web_framework.rest_view import RestWheelSet, HttpResponse
from web_framework.wsgi_switch import WsgiSwitch

error_router = ErrorTypeSwitch(default=SequentialSwitch())
router = WsgiSwitch(error_track=error_router)

@dataclass
class User:
    id: int
    name: str

@dataclass
class Error:
    error: str

class UserManager:
    def __init__(self):
        self.users = []

class ManagerJoint(Joint):
    def __init__(self, track: Track):
        super().__init__(track)
        self.user_manager = UserManager()

    def __call__(self, event, context: WsgiContext):
        context.data["user_manager"] = self.user_manager
        return self.track(event, context)

@router.track("/", methods=["GET"])
def index(environ, context: WsgiContext) -> HttpResponse[dict[str, str]]:
    return HttpResponse(
        body={"ok": "Index"}
    )

@router.track("/users", methods=["GET"])
def get_users(environ, context: WsgiContext) -> HttpResponse[list[User]]:
    user_manager: UserManager = context.data["user_manager"]
    return HttpResponse(body=user_manager.users)

@router.track("/users", methods=["POST"])
def add_user(environ, context: WsgiContext) -> HttpResponse[User]:
    user_manager: UserManager = context.data["user_manager"]

    user = User(
        id=len(user_manager.users),
        name=f"User {len(user_manager.users)}",
    )

    user_manager.users.append(user)
    return HttpResponse(body=user)

@error_router.track(RouteNotFound)
def handle_not_found_error(
        environ: ErrorEvent, context: WsgiContext,
) -> HttpResponse[Error]:
    logging.error(f"Resource {environ.event['PATH_INFO']} not found")
    return HttpResponse(
        status=HTTPStatus.NOT_FOUND,
        body=Error(error=f"Resource {environ.event['PATH_INFO']} not found"),
    )

@error_router.default.track()
def handle_any_error(
        environ: ErrorEvent, context: WsgiContext,
) -> HttpResponse[Error]:
    logging.error("Unhandled error in HTTP")
    return HttpResponse(
        status=HTTPStatus.INTERNAL_SERVER_ERROR,
        body=Error(error=str(environ.exception)),
    )

router = ManagerJoint(router)
app = App(RestWheelSet(router))


import keras
import numpy as np
from keras.layers import Input, Dense
from keras.models import Model
from keras.optimizers import Adam
from sklearn.cluster import KMeans
from keras.models import load_model
import csv
import sys

path_train = sys.argv[1]
path_test = sys.argv[2]
path_out = sys.argv[3]

def load_data():
    with open(path_test, 'r') as file:
        reader = csv.reader(file, delimiter=',')
        test_x = []
        iter = 0
        for line in reader:
            if iter == 0:
                iter += 1
                continue
            test = []
            test.append(int(line[1]))
            test.append(int(line[2]))
            test_x.append(test)
            iter += 1
        test_x = np.array(test_x)
        print(test_x)
        return test_x

def build_model():
    input_img = Input(shape=(784, ))
    encoded = Dense(128, activation='relu')(input_img)
    encoded = Dense(64, activation='relu')(encoded)
    encoded = Dense(32, activation='relu')(encoded)

    decoded = Dense(64, activation='relu')(encoded)
    decoded = Dense(128, activation='relu')(decoded)
    decoded = Dense(784, activation='sigmoid')(decoded)

    encoder = Model(input = input_img, output=encoded)

    adam = Adam(lr=5e-4)
    autoencoder = Model(input=input_img, output=decoded)
    autoencoder.compile(optimizer=adam, loss='mse')
    autoencoder.summary()
    return encoder, autoencoder

def main():
    x = np.load(path_train)
    x = x.astype('float32') / 255.
    #train_num = 130000
    #print(x.shape)
    #train_x = x[:train_num]
    #valid_x = x[train_num:]

    """encoder, autoencoder = build_model()
    autoencoder.fit(train_x, train_x, epochs=1000, batch_size=256, shuffle=True, validation_data=(valid_x, valid_x))
    autoencoder.save('autoencoder.h5')
    encoder.save('encoder.h5')"""
    encoder = load_model('encoder.h5')
    encoder_imgs = encoder.predict(x)
    encoder_imgs = encoder_imgs.reshape(encoder_imgs.shape[0], -1)
    kmeans = KMeans(n_clusters=2, random_state=0).fit(encoder_imgs)

    test_x = load_data()
    same = []
    test_y = np.zeros(shape=test_x.shape)
    for i in range(test_x.shape[0]):
        test_y[i][0] = i
        a = kmeans.labels_[test_x[i][0]]
        b = kmeans.labels_[test_x[i][1]]
        if a == b:
            test_y[i][1] = 1
        else:
            test_y[i][1] = 0

    with open(path_out, 'w') as file:
        writer = csv.writer(file, delimiter=',')
        writer.writerow(['ID', 'Ans'])
        for i in range(test_y.shape[0]):
            writer.writerow([int(test_y[i][0]), int(test_y[i][1])])

if __name__ == "__main__":
    main()


from kafka import KafkaProducer
import sys

msg = str(sys.argv[1])

def run():

    try:

        producer = KafkaProducer(
            bootstrap_servers = ['localhost:9093','localhost:9094','localhost:9095']
        )

        def message() -> dict:

            if msg[0] < "N":
                partition = 0
            else:
                partition = 1

            return{
                "value" : msg,
                "partition" : partition
            }

        message_sent = message()

        result = producer.send(
            "Users",
            bytes(str(msg), 'utf-8')
        )

        print("Done! Created Successfully! " + str(message_sent))
        producer.flush()

        producer.close()

    except Exception as e:
        print("Something bad happened " + str(e))

    finally:
        sys.exit()

run()


import arviz as az

import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import pymc as pm
import pytensor
import pytensor.tensor as pt

print(f"Running on PyMC v{pm.__version__}")
def my_model(theta, x):
    m, c = theta
    return m * x + c

def my_loglike(theta, x, data, sigma):
    model = my_model(theta, x)
    model_2 = my_model(theta, x)
    lh_1=-(0.5 / sigma**2) * np.sum((data - model) ** 2)
    lh_2=-(0.5 / sigma**2) * np.sum((data - model) ** 2)
    return lh_1+lh_2
# define a pytensor Op for our likelihood function
class LogLike(pt.Op):

    """
    Specify what type of object will be passed and returned to the Op when it is
    called. In our case we will be passing it a vector of values (the parameters
    that define our model) and returning a single "scalar" value (the
    log-likelihood)
    """

    itypes = [pt.dvector]  # expects a vector of parameter values when called
    otypes = [pt.dscalar]  # outputs a single scalar value (the log likelihood)

    def __init__(self, loglike, data, x, sigma):
        """
        Initialise the Op with various things that our log-likelihood function
        requires. Below are the things that are needed in this particular
        example.

        Parameters
        ----------
        loglike:
            The log-likelihood (or whatever) function we've defined
        data:
            The "observed" data that our log-likelihood function takes in
        x:
            The dependent variable (aka 'x') that our model requires
        sigma:
            The noise standard deviation that our function requires.
        """

        # add inputs as class attributes
        self.likelihood = loglike
        self.data = data
        self.x = x
        self.sigma = sigma

    def perform(self, node, inputs, outputs):

        # the method that is used when calling the Op
        (theta,) = inputs  # this will contain my variables

        # call the log-likelihood function
        logl = self.likelihood(theta, self.x, self.data, self.sigma)
        outputs[0][0] = np.array(logl)  # output the log-likelihood

# set up our data
N = 10  # number of data points
sigma = 1.0  # standard deviation of noise
x = np.linspace(0.0, 9.0, N)

mtrue = 0.4  # true gradient
ctrue = 3.0  # true y-intercept

truemodel = my_model([mtrue, ctrue], x)

# make data
rng = np.random.default_rng(716743)
data = sigma * rng.normal(size=N) + truemodel

# create our Op
logl = LogLike(my_loglike, data, x, sigma)

# use PyMC to sampler from log-likelihood
for j in range(0, 1):
    with pm.Model():
        # uniform priors on m and c
        #m = pm.Uniform("m", lower=-10.0, upper=10.0)
        #c = pm.Uniform("c", lower=-10.0, upper=10.0)
        #print(vars(m))
        v=[pm.Uniform(x, lower=-10, upper=10) for x in ["m"]]
        q=[pm.Uniform(x, lower=-10, upper=10) for x in ["c"]]
        # convert m and c to a tensor vector
        theta = pt.as_tensor_variable(v+q)
        save_str= "/home/henney/Documents/Oxford/General_electrochemistry/heuristics/file_{0}.nc".format(j)

        # use a Potential to "call" the Op and include it in the logp computation
        pm.Potential("likelihood", logl(theta))

        # Use custom number of draws to replace the HMC based defaults
        idata_mh = pm.sample(3000, tune=1000)
        idata_mh.to_netcdf(save_str)

# plot the traces
az.plot_trace(idata_mh, lines=[("m", {}, mtrue), ("c", {}, ctrue)])