import collections
import re

from google.cloud import bigquery

client = bigquery.Client()

VOCABULARY_DATASET_RE = re.compile(r'^vocabulary\d{8}$')
RDR_DATASET_RE = re.compile(r'^rdr\d{8}$')
UNIONED_DATASET_RE = re.compile(r'^\d{4}q\dr\d_unioned_ehr$')
COMBINED_DATASET_RE = re.compile(r'^\d{4}q\dr\d_combined$')
DEID_DATASET_RE = re.compile(r'^R\d{4}q\dr\d_deid(?:_clean)$')
TREND_N = 3

def is_vocabulary_dataset(dataset_id):
    return re.match(VOCABULARY_DATASET_RE, dataset_id) is not None

def is_rdr_dataset(dataset_id):
    return re.match(RDR_DATASET_RE, dataset_id) is not None

def is_unioned_dataset(dataset_id):
    return re.match(UNIONED_DATASET_RE, dataset_id) is not None

def is_combined_dataset(dataset_id):
    return re.match(COMBINED_DATASET_RE, dataset_id) is not None

def is_deid_dataset(dataset_id):
    return re.match(DEID_DATASET_RE, dataset_id) is not None

def _datasets():
    DefaultDatasets = collections.namedtuple('DefaultDatasets', 'latest trend')
    dataset_list = list(client.list_datasets())
    dataset_list.sort(key=lambda d: d.dataset_id, reverse=True)
    vocabulary = []
    rdr = []
    unioned = []
    combined = []
    deid = []

    for dataset in dataset_list:
        dataset_id = dataset.dataset_id
        if is_vocabulary_dataset(dataset_id):
            vocabulary.append(dataset_id)
        elif is_rdr_dataset(dataset_id):
            rdr.append(dataset_id)
        elif is_unioned_dataset(dataset_id):
            unioned.append(dataset_id)
        elif is_combined_dataset(dataset_id):
            combined.append(dataset_id)
        elif is_deid_dataset(dataset_id):
            deid.append(dataset_id)

    LatestDatasets = collections.namedtuple(
        'LatestDatasets', 'vocabulary rdr unioned combined deid')
    latest = LatestDatasets(vocabulary=vocabulary[0],
                            rdr=rdr[0] if rdr else [],
                            unioned=unioned[0] if unioned else [],
                            combined=combined[0] if combined else [],
                            deid=deid[0] if deid else [])

    TrendDatasets = collections.namedtuple('TrendDatasets',
                                           'rdr unioned combined deid')
    trend = TrendDatasets(rdr=rdr[0:TREND_N],
                          unioned=unioned[0:TREND_N],
                          combined=combined[0:TREND_N],
                          deid=deid[0:TREND_N])

    return DefaultDatasets(latest=latest, trend=trend)

DEFAULT_DATASETS = _datasets()


from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.keys import Keys
from pyvirtualdisplay import Display
import os
import time

def post_facebook(text, img):
    # Open browser in virtual display and hide it, open facebook
    # Disable notifications
    Display(visible=0, size=(800, 600)).start()
    opts = webdriver.ChromeOptions()
    prefs = {"profile.default_content_setting_values.notifications": 2}
    opts.add_experimental_option("prefs", prefs)
    driver = webdriver.Chrome(options=opts)
    driver.get('https://www.facebook.com')

    # Type user credentials
    driver.find_element_by_id('email').send_keys('asalgado21@gmail.com')
    driver.find_element_by_id('pass').send_keys('carmar1971?', Keys.ENTER)

    # Go to page
    driver.find_element_by_id('logoutMenu').click()
    driver.find_element_by_xpath('//div[text()="Salud y pimienta"]').click()
    time.sleep(10)

    # Write post
    xpath = '//div[text()="Escribe una publicación..."]'
    d = driver.find_element_by_xpath(xpath)
    driver.execute_script("arguments[0].click();", d)

    actions = ActionChains(driver)
    actions.send_keys(text)
    actions.perform()

    # Add Image
    driver.find_element_by_xpath('//div[text()="Foto/video"]').click()
    time.sleep(10)

    xpath = '//input[@data-testid="media-attachment-add-photo"]'
    driver.find_element_by_xpath(xpath).send_keys(os.getcwd()+"/"+img)
    time.sleep(10)

    # Submit post
    driver.find_element_by_xpath('//span[text()="Compartir"]').click()


import subprocess
import glob
import os
import random
import sys
from typing import Tuple

def test(array: list, executable_path: str) -> Tuple[bool, list, list]:
    """
    Function to test

    Parameters
    ----------
    array : Array
        list
    executable_path : Path to execution file
        str

    Returns
    -------
    True/False and actual, expected arrays
    """
    args = str(len(array)) + "\n"
    args += "\n".join(str(x) for x in array)

    res = subprocess.run(
        [executable_path], input=args.encode(), check=True, capture_output=True
    )

    result = [int(x) for x in res.stdout.decode().split(" ")[:-1]]

    return result == sorted(array), result, sorted(array)

def get_random_array(num: int) -> list:
    """
    Function to get random array

    Parameters
    ----------
    num : Number of elements in array
        int

    Returns
    -------
    Random Array
    """
    random_array = [random.randint(1, 10000) for _ in range(num)]

    return random_array

if __name__ == "__main__":
    directory_path = "build/task3"

    all_files = os.listdir(directory_path)

    executable_files = [
        file
        for file in all_files
        if os.stat(os.path.join(directory_path, file)).st_mode & 0o100 != 0
        and file != "CMakeFiles"
    ]

    for executable_file in executable_files:
        ex = "./" + directory_path + "/" + executable_file
        for _ in range(5):
            length = random.randint(1, 100)
            res, actual, expected = test(get_random_array(length), ex)
            if not res:
                print("Wrong Answer")
                print("Actual: ", actual)
                print("Expected: ", expected)
                sys.exit(1)

    print("All Done!!!")


from django.urls import path, include
from . import views

urlpatterns = [
    path('medicalstaff/', views.medicalstaff, name='medicalstaff'),
    path('profile/', views.user_profile, name='user_profile'),
    path('userupdate/', views.user_update, name='user_update'),

    # API Routes
    path('getprofile/', views.getprofile, name='getprofile'),
    path('getpersonal/', views.getpersonal, name='getpersonal'),
    path('employees/', views.employees, name='employees'),
    path('getprofession/<int:id>', views.getprofession, name='getprofession'),
]


import numpy as np
import os
from os.path import join
import sys
from skimage.io import imread

def data_download(id, tissue, TMC):
	img_dir = join(data_dir, TMC, tissue, id)
	if not os.path.exists(img_dir):
		os.makedirs(img_dir)
	BASE_URL = 'https://g-d00e7b.09193a.5898.dn.glob.us/'

	def download_files(prefix, img_dir, id):
		file_names = [
			'cell_centers.csv',
			'img_binary.png',
			'nuclei_channel_total.csv',
			'cell_channel_total.csv',
			'cell_boundaries_channel_total.csv',
			'nucleus_boundaries_channel_total.csv'
		]

		for file_name in file_names:
			if file_name != 'img_binary.png':
				output_file = join(img_dir, f'reg1_stitched_expressions.ome.tiff-{file_name}')
				download_url = f'{BASE_URL}{id}/sprm_outputs/{prefix}-{file_name}?download=1'
			else:
				output_file = join(img_dir, f'reg1_stitched_expressions.ome.tiff_{file_name}')
				download_url = f'{BASE_URL}{id}/sprm_outputs/{prefix}_{file_name}?download=1'
			os.system(f'wget -O {output_file} {download_url}')

	if TMC == 'Florida':
		download_files('reg1_stitched_expressions.ome.tiff', img_dir, id)

		# Check if the binary image is empty and if so, re-download with a different prefix
		if os.stat(join(img_dir, 'reg1_stitched_expressions.ome.tiff_img_binary.png')).st_size == 0:
			download_files('reg001_expr.ome.tiff', img_dir, id)
	else:
		download_files('reg001_expr.ome.tiff', img_dir, id)

	image = imread(join(img_dir,'reg1_stitched_expressions.ome.tiff_img_binary.png'))
	np.savetxt(join(img_dir,'reg1_stitched_expressions.ome.tiff-img_shape.txt'), image.shape, fmt='%i')

if __name__ == '__main__':
	script_dir = os.path.dirname(os.path.abspath(__file__))
	tissue_list = ['LN', 'LI', 'SI', 'SPLEEN', 'THYMUS']
	data_dir = sys.argv[1]
	for tissue in tissue_list:
		if tissue == 'LI' or tissue == 'SI':
			TMC = 'Stanford'
		else:
			TMC = 'Florida'
		checklist = np.loadtxt(join(script_dir, 'checklist_CODEX_PPM_' + tissue + '_new.txt'), dtype=str)
		img_num = checklist.shape[0]
		for idx in range(img_num):
			data_download(checklist[idx], tissue, TMC)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('patients', '0029_profilesuser_bp_interval'),
    ]

    operations = [
        migrations.RemoveField(
            model_name='profilesuser',
            name='id_blood',
        ),
    ]


# -*- coding: utf-8 -*-
import time
from datetime import datetime
from odoo import fields, models, api, _
from odoo.exceptions import except_orm
from odoo import exceptions
from odoo.exceptions import UserError, ValidationError

class HRSalaryAdvance(models.Model):
    _name = "hr.salary.advance"
    _description = 'Employee Advance Salary'
    _inherit = ['mail.thread', 'mail.activity.mixin']

    name = fields.Char(string='Reference', required=True, copy=False, readonly=True, index=True, default=lambda self: _('New'))

    employee_id = fields.Many2one('hr.employee', string='Employee', required=True, readonly=True, states={'draft': [('readonly', False)]},)
    department_id = fields.Many2one('hr.department', string='Department', related='employee_id.department_id')
#     job_title = fields.Char(related='employee_id.job_title')
    employee_contract_id = fields.Many2one('hr.contract', string='Contract', required=True, readonly=True, states={'draft': [('readonly', False)]}, domain="[('employee_id','=',employee_id)]")
    date = fields.Date(string='Request Date', required=True, readonly=True, states={'draft': [('readonly', False)]}, default=lambda self: fields.Date.today())
    reason = fields.Text(string='Reason')
    currency_id = fields.Many2one('res.currency', string='Currency', required=True,
                                  default=lambda self: self.env.user.company_id.currency_id)
    company_id = fields.Many2one('res.company', string='Company', required=True, readonly=True, states={'draft': [('readonly', False)]}, default=lambda self: self.env.user.company_id)
    user_id = fields.Many2one('res.users', string='Requester', readonly=True, states={'draft': [('readonly', False)]}, tracking=True, default=lambda self: self.env.user)

    amount = fields.Monetary(string='Request Amount', required=True, readonly=True, states={'draft': [('readonly', False)]},)
    exceed_condition = fields.Boolean(string='Exceed than maximum', help="The Advance is greater than the maximum percentage in salary structure")
    state = fields.Selection([('draft', 'Draft'),
                              ('confirm', 'Confirmed'),
                              ('approval1', 'Approved by Manager'),
                              ('approval2', 'Approved by HR'),
                              ('approval3', 'Approved by Director'),
                              ('paid', 'Paid'),
                              ('cancel', 'Reject'),
                              ('done', 'Done'),], string='Status', default='draft', track_visibility='onchange')

    partner_id = fields.Many2one('res.partner', 'Employee Partner', readonly=False, states={'paid': [('readonly', True)]},)
    journal_id = fields.Many2one('account.journal', string='Payment Method', domain="[('type','in',['cash','bank'])]")
    payment_id = fields.Many2one('account.payment', string='Payment', readonly=True, compute='_get_payment')
    payment_amount = fields.Monetary(related='payment_id.amount')

    @api.depends('state')
    def _get_payment(self):
        payment_id = self.env['account.payment'].search([('hr_salary_advance_id','=',self.id)],limit=1)
        self.payment_id = payment_id.id
        if not payment_id:
            self.payment_id = False

    @api.constrains('amount')
    def _check_amount(self):
        if self.amount:
            contract_id = self.env['hr.contract'].search([('employee_id','=', self.employee_id.id),('state','=','open')], limit=1)
            self.employee_contract_id = contract_id.id
            adv = self.amount
            amt = (self.employee_contract_id.structure_type_id.max_percent * self.employee_contract_id.wage) / 100
            if adv > amt and not self.exceed_condition:
                raise UserError(('Error!', 'Advance amount is greater than allotted. You are only allow to enter Amount '+str(amt)))

    def unlink(self):
        if any(self.filtered(lambda loan: loan.state not in ('draft', 'cancel'))):
            raise UserError(_('You cannot delete a Request which is not draft or cancelled!'))
        return super(HRSalaryAdvance, self).unlink()

    @api.onchange('employee_id')
    def onchange_employee_id(self):
        if self.employee_id:
            contract_id = self.env['hr.contract'].search([('employee_id','=',self.employee_id.id),('state','=','open')])
            self.employee_contract_id = contract_id.id
            self.partner_id = self.employee_id.address_home_id.id

    @api.model
    def create(self, vals):
        if 'company_id' in vals:
            self = self.with_company(vals['company_id'])
        if vals.get('name', _('New')) == _('New'):
            seq_date = None
            if 'date' in vals:
                seq_date = fields.Datetime.context_timestamp(self, fields.Datetime.to_datetime(vals['date']))
            vals['name'] = self.env['ir.sequence'].next_by_code('hr.salary.advance', sequence_date=seq_date) or _('New')
        result = super(HRSalaryAdvance, self).create(vals)
        return result

    def action_confirm(self):
        self.state = 'confirm'

    def action_approval1(self):
        """This Approve the employee salary advance request.
                   """
        emp_obj = self.env['hr.employee']
        address = emp_obj.browse([self.employee_id.id]).address_home_id

        salary_advance_search = self.search([('employee_id', '=', self.employee_id.id), ('id', '!=', self.id),
                                             ('state', '=', 'approve')])
        current_month = datetime.strptime(str(self.date), '%Y-%m-%d').date().month
        for each_advance in salary_advance_search:
            existing_month = datetime.strptime(str(each_advance.date), '%Y-%m-%d').date().month
            if current_month == existing_month:
                raise UserError(('Error!', 'Advance can be requested once in a month'))

        salary_advance_search = self.search([('employee_id', '=', self.employee_id.id), ('id', '!=', self.id), ('id', '!=', self.id),
                                             ('state', '=', 'approve')])
        current_year = datetime.strptime(str(self.date), '%Y-%m-%d').date().year
        for each_advance in salary_advance_search:
            existing_year = datetime.strptime(str(each_advance.date), '%Y-%m-%d').date().year

        struct_id = self.employee_id.contract_id.structure_type_id
        contract = self.env['hr.contract'].search([('state','=','open'),('employee_id','=',self.employee_id.id)])
        if(self.employee_contract_id.id == False):
            self.employee_contract_id = contract
        if not struct_id.max_percent or not struct_id.advance_date:
            raise UserError(('Error!', 'Max percentage or advance days are not provided in Contract'))
        adv = self.amount
        amt = (self.employee_contract_id.structure_type_id.max_percent * self.employee_contract_id.wage) / 100
        if adv > amt and not self.exceed_condition:
            raise UserError(('Error!', 'Advance amount is greater than allotted'))

        if not self.amount:
            raise UserError(('Warning', 'You must Enter the Salary Advance amount'))
        payslip_obj = self.env['hr.payslip'].search([('employee_id', '=', self.employee_id.id),
                                                     ('state', '=', 'done'), ('date_from', '<=', self.date),
                                                     ('date_to', '>=', self.date)])
        if payslip_obj:
            raise UserError(('Warning', "This month salary already calculated"))

        for slip in self.env['hr.payslip'].search([('employee_id', '=', self.employee_id.id)]):
            slip_moth = datetime.strptime(str(slip.date_from), '%Y-%m-%d').date().month
            if current_month == slip_moth + 1:
                slip_day = datetime.strptime(str(slip.date_from), '%Y-%m-%d').date().day
                current_day = datetime.strptime(str(self.date), '%Y-%m-%d').date().day
                if current_day - slip_day < struct_id.advance_date:
                    raise exceptions.Warning(
                        _('Request can be done after "%s" Days From prevoius month salary') % struct_id.advance_date)
        self.state = 'approval1'

    def action_approval2(self):
        self.state = 'approval2'

    def action_approval3(self):
        """This Approve the employee salary advance request from accounting department.
                   """
        emp_obj = self.env['hr.employee']
        address = emp_obj.browse([self.employee_id.id]).address_home_id

        salary_advance_search = self.search([('employee_id', '=', self.employee_id.id), ('id', '!=', self.id),
                                             ('state', '=', 'approve')])
        current_month = datetime.strptime(str(self.date), '%Y-%m-%d').date().month
        for each_advance in salary_advance_search:
            existing_month = datetime.strptime(str(each_advance.date), '%Y-%m-%d').date().month
            if current_month == existing_month:
                raise UserError(('Error!', 'Advance can be requested once in a month'))

        salary_advance_search = self.search([('employee_id', '=', self.employee_id.id), ('id', '!=', self.id), ('id', '!=', self.id),
                                             ('state', '=', 'approve')])
        current_year = datetime.strptime(str(self.date), '%Y-%m-%d').date().year
        for each_advance in salary_advance_search:
            existing_year = datetime.strptime(str(each_advance.date), '%Y-%m-%d').date().year

        struct_id = self.employee_id.contract_id.structure_type_id
        contract = self.env['hr.contract'].search([('state','=','open'),('employee_id','=',self.employee_id.id)])
        if(self.employee_contract_id.id == False):
            self.employee_contract_id = contract
        if not struct_id.max_percent or not struct_id.advance_date:
            raise UserError(('Error!', 'Max percentage or advance days are not provided in Contract'))
        adv = self.amount
        amt = (self.employee_contract_id.structure_type_id.max_percent * self.employee_contract_id.wage) / 100
        if adv > amt and not self.exceed_condition:
            raise UserError(('Error!', 'Advance amount is greater than allotted'))

        if not self.amount:
            raise UserError(('Warning', 'You must Enter the Salary Advance amount'))
        payslip_obj = self.env['hr.payslip'].search([('employee_id', '=', self.employee_id.id),
                                                     ('state', '=', 'done'), ('date_from', '<=', self.date),
                                                     ('date_to', '>=', self.date)])
        if payslip_obj:
            raise UserError(('Warning', "This month salary already calculated"))

        for slip in self.env['hr.payslip'].search([('employee_id', '=', self.employee_id.id)]):
            slip_moth = datetime.strptime(str(slip.date_from), '%Y-%m-%d').date().month
            if current_month == slip_moth + 1:
                slip_day = datetime.strptime(str(slip.date_from), '%Y-%m-%d').date().day
                current_day = datetime.strptime(str(self.date), '%Y-%m-%d').date().day
                if current_day - slip_day < struct_id.advance_date:
                    raise exceptions.Warning(
                        _('Request can be done after "%s" Days From prevoius month salary') % struct_id.advance_date)
        if not self.amount:
            raise UserError(('Warning', 'You must Enter the Salary Advance amount'))

        self.state = 'approval3'
        return True

    def action_cancel(self):
        self.state = 'cancel'

    def action_done(self):
        self.state = 'done'

    def action_payment(self):
        payment = self.env['account.payment']
        payment.create({
            'payment_type': 'outbound',
            'partner_type': 'supplier',
            'partner_id': self.partner_id.id,
            'company_id': self.company_id.id,
            'amount': self.amount,
            'currency_id': self.currency_id.id,
            'journal_id': self.journal_id.id,
            'date': fields.Date.today(),
            'ref': self.name,
            'hr_salary_advance_id': self.id
        })
        self.update({
            'payment_id': payment.id,
            'state': 'paid'
        })
        return payment

#


import pandas as pd
import matplotlib.pyplot as plt

def plot(path, name):
    df = pd.read_csv(path["data_folder"] + "robotSensors", index_col=False, sep=" ", header=None, names=["t", "light", "bluelight", "redlight", "groundmemory", "redbattery"])

    fig, ax = plt.subplots(2)
    df.plot(x="t", y="light", color="tab:orange", ax=ax[0])
    df.plot(x="t", y="bluelight", color="tab:blue", ax=ax[0])
    df.plot(x="t", y="groundmemory", color="tab:brown", ax=ax[0])

    df.plot(x="t", y="redlight", color="tab:red", ax=ax[1])
    df.plot(x="t", y="redbattery", color="tab:green", ax=ax[1])
    plt.savefig(path["exp_folder"] + name, bbox_inches='tight')


import serial
from config import CONFIG
import logging
import time

class DisplayIO:

    def connect(self):
        try:
            ser = serial.Serial(CONFIG.SERIALPORT, CONFIG.SERIALRATE, timeout=CONFIG.SERIALTIMEOUT)
            ser.flush()
            msg = f"connect(): Connected to {ser}"
            logging.debug(msg)
            return ser
        except Exception as error:
            msg = f"connect(): Could not connect to serial port {CONFIG.SERIALPORT}"
            logging.error(msg)
            logging.debug(error)
            return 0

    def getserialdata(self, serial_connection):
        try:
            data = serial_connection.readline().decode('utf-8').rstrip()
            msg = f"getserialdata(): String received: {data}"
            logging.debug(msg)
            return data
        except Exception as error:
            msg = f"connect(): Error receiving serial data!"
            logging.error(msg)
            logging.debug(error)
            return 0

    def sendserialdata(self, serial_connection, data):
        send = serial_connection.write(data.encode('utf-8'))
        time.sleep(0.1)
        msg = f"sendserialdata(): Data sent: {send}"
        logging.debug(msg)
        return send


import copy
import sys
from collections import deque

sys.stdin = open("./input.txt", "r")

N, Q = map(int,sys.stdin.readline().split())
NN = pow(2,N)
board = [list(map(int, sys.stdin.readline().split())) for _ in range(NN)]
L = list(map(int, sys.stdin.readline().split()))

dx, dy = (-1,1,0,0),(0,0,-1,1)

print(board)

def count_cluster():
    visited = [[False]*NN for _ in range(NN)]
    cluster = {}
    queue = deque()
    clusters = 1
    sum = 0
    for i in range(NN):
        for j in range(NN):
            sum += board[i][j]
            if not visited[i][j] and board[i][j] > 0: # 방문안했고, 0보다 크다.
                queue.append((i,j))
                while queue:
                    x, y = queue.popleft()
                    if not visited[x][y] and board[x][y] > 0:
                        visited[x][y] = True
                        for d in range(4):
                            nx, ny = x+dx[d], y+dy[d]
                            if nx < 0 or nx >= NN or ny <0 or ny >= NN or board[nx][ny] <= 0: continue
                            queue.append((nx,ny))
                        if board[x][y] > 0:
                            if clusters not in cluster.keys():
                                cluster[clusters] = [(x, y)]
                            else:
                                cluster[clusters].append((x, y))
                clusters+=1
    max_cluster = 0
    for k, c in cluster.items():
        if max_cluster < len(c):
            max_cluster = len(c)
    print(sum)
    print(max_cluster)
##
def check_board():
    melt_ice = deque()
    for i in range(NN):
        for j in range(NN):
            if board[i][j] > 0: # 방문안했고, 0보다 크다.
                cnt = 0
                for d in range(4):
                    nx, ny = i+dx[d], j+dy[d]
                    if nx < 0 or nx >= NN or ny <0 or ny >= NN or board[nx][ny] <= 0: continue
                    cnt += 1
                if not cnt >= 3:
                    print('meltice : ', i,j, cnt)
                    melt_ice.append((i,j))
    while melt_ice:
        x,y = melt_ice.popleft()
        board[x][y] -= 1

    print('\n=============================================================')
    for _ in board:
        print(_)
    print('=============================================================\n')

def tornado(start, end, region):
    box = deque()
    N = end - start
    for i in range(N):
        box.append(region[start][start + i])
        box.append(region[start + i][end])
        box.append(region[end - i][start])
        box.append(region[end][end - i])
        region[start + i][end] = box.popleft()
        region[end][end - i] = box.popleft()
        region[start][start + i] = box.popleft()
        region[end - i][start] = box.popleft()
    if N > 2:
        return tornado(start + 1, end - 1, region)
    else:
        return region

def firestorm(level):
    grid = pow(2, level)
    print(grid, int(NN/grid), NN)
    box = [[0] * grid for _ in range(grid)]
    for g in range(0, NN, grid):
        print(g)
        for j in range(int(NN/grid)):
            for i in range(grid):
                box[i] = board[g+i][(grid*j):(grid*j)+grid]
            result = tornado(0, grid-1, box)
            for i in range(grid):
                board[g+i][(grid*j):(grid*j)+grid] = result[i]
    check_board()

for level in L:
    firestorm(level)

count_cluster()

20058_마법사_상어와_파이어스톰.py


import requests

from rest_framework.response import Response
from rest_framework.views import APIView
from rest_framework.pagination import LimitOffsetPagination
from rest_framework import status

from drf_yasg.utils import swagger_auto_schema
from drf_yasg import openapi

from .serializers import ClimateChangeInfo, ClimateChangeInfoSerializer

# Create your views here.

# To ensure data-type consistency, type check is added,
# if failed then by default 0 will be assigned
def custom_float(num):
    try:
        if type(eval(num)) == float:
            return float(num)
        elif type(eval(num)) == int:
            return int(num)
    except Exception as e:
        return 0

# Parsing climate change data
def parse_weather_data(region, parameter):
    try:
        url = 'https://www.metoffice.gov.uk/pub/data/weather/uk/climate/datasets/{0}/date/{1}.txt'.format(parameter, region)
        response = requests.get(url)
        data = response.text.split('\n')
        last_updated_date = ' '.join(data[4].split()[-2:])
        data_points = []
        for row in data[6:-1]:
            data_points.append(list(map(custom_float, row.split())))
        body = {
            'last_updated_date': last_updated_date,
            'data_points': data_points
        }
        return body
    except Exception as e:
        return {'error': 'Invalid region, parameter combination'}

test_param = [
    openapi.Parameter('region', openapi.IN_QUERY, description="Enter region name", type=openapi.TYPE_STRING),
    openapi.Parameter('parameter', openapi.IN_QUERY, description="Enter parameter name", type=openapi.TYPE_STRING),
    openapi.Parameter('limit', openapi.IN_QUERY, description="Enter limit value", type=openapi.TYPE_INTEGER),
    openapi.Parameter('offset', openapi.IN_QUERY, description="Enter offset value", type=openapi.TYPE_INTEGER),
]

class GetClimateChangeDataView(APIView):
    @swagger_auto_schema(
        responses={
            405: openapi.Response('Method Not Allowed', openapi.Schema(type=openapi.TYPE_OBJECT)),
        }
    )
    def post(self, request):
        return Response({'error' : 'Invalid Request Method'}, status=status.HTTP_405_METHOD_NOT_ALLOWED)
    @swagger_auto_schema(
        manual_parameters=test_param,
        responses={
            200: ClimateChangeInfoSerializer(many=True),
            400: openapi.Response('Bad Request', openapi.Schema(type=openapi.TYPE_OBJECT)),
        }
    )
    def get(self, request):
        paginator = LimitOffsetPagination()
        response = parse_weather_data(request.GET['region'], request.GET['parameter'])
        if 'error' in response:
            return Response(response, status=status.HTTP_400_BAD_REQUEST)
        data_points = response['data_points']
        climate_change_info_list = []
        for data in data_points:
            climate_change_object = ClimateChangeInfo(
                year = data[0],
                jan = data[1],
                feb = data[2],
                mar = data[3],
                apr = data[4],
                may = data[5],
                jun = data[6],
                jul = data[7],
                aug = data[8],
                sep = data[9],
                oct = data[10],
                nov = data[11],
                dec = data[12],
                win = data[13],
                spr = data[14],
                sum = data[15],
                aut = data[16],
                ann = data[17]
            )
            climate_change_info_list.append(climate_change_object)
        # the many param informs the serializer that it will be serializing more than a single climate change object.
        query_set = climate_change_info_list
        context = paginator.paginate_queryset(query_set, request)
        serializer = ClimateChangeInfoSerializer(context, many=True)
        return paginator.get_paginated_response(serializer.data)

if __name__ == '__main__':
    parse_weather_data('England_SE_and_Central_S', 'Rainfall')


import logging
import os
import re
import threading
from typing import List, Optional, Dict

from injector import singleton, inject

from gst.model.monitored_item import MonitoredItem
from gst.model.processor import Processor
from gst.model.system_info import SystemInfo
from gst.util.concurrency import synchronized_with_attr
from gst.util.sensors import FeatureType

_LOG = logging.getLogger(__name__)
PATH_PROC_CPUINFO = '/proc/cpuinfo'
CLEAN_CPU_STRING_REGEX = r"chipset|company|components|computing|computer|corporation|communications|electronics|" \
                         r"electrical|electric|gmbh|group|incorporation|industrial|international|\bnee\b|revision" \
                         r"|semiconductor|software|technologies|technology|ltd\.|<ltd>|\bltd\b|inc\.|<inc>|\binc\b" \
                         r"|intl\.|co\.|<co>|corp\.|<corp>|\(tm\)|\(r\)|®|\(rev ..\)|\'|\"|\sinc\s*$|@|cpu |cpu deca" \
                         r"|([0-9]+|single|dual|two|triple|three|tri|quad|four|penta|five|hepta|six|hexa|seven|octa" \
                         r"|eight|multi)[ -]core|ennea|genuine|multi|processor|single|triple|[0-9\.]+ *[MmGg][Hh][Zz]"

@singleton
class ProcCpuinfoRepository:
    @inject
    def __init__(self) -> None:
        self._lock = threading.RLock()

    cpu_info_mapping = {
        'processor_id': [int, 'processor'],
        'vendor_id': [str, 'vendor_id'],
        'specification': [str, 'model name'],
        'family': [int, 'cpu family'],
        'model': [int, 'model'],
        'stepping': [int, 'stepping'],
        'microcode': [str, 'microcode'],
        'core_speed': [float, 'cpu MHz', 'cpu speed', 'clock'],
        'physical_package_id': [int, 'physical id'],
        'threads': [int, 'siblings'],
        'core_id': [int, 'core id'],
        'cores': [int, 'cpu cores'],
        'flags': [str, 'flags'],
        'bugs': [str, 'bugs'],
        'bogomips': [float, 'bogomips'],
    }

    @synchronized_with_attr("_lock")
    def refresh(self, system_info: SystemInfo) -> SystemInfo:
        if not os.path.exists(PATH_PROC_CPUINFO):
            _LOG.warning("%s not found", PATH_PROC_CPUINFO)
            return system_info

        try:
            with open(PATH_PROC_CPUINFO, 'r') as file:
                output = file.read()
        except IOError:
            _LOG.exception("Error while reading %s", PATH_PROC_CPUINFO)
            return system_info

        # load data on a temp variable
        temp_processor_list: List[Processor] = []
        tmp_proc: Optional[Processor] = None
        for line in output.splitlines():
            if ':' in line:
                label, value = line.split(':', 1)
                label = label.strip()
                value = value.strip()
                if label in self.cpu_info_mapping['processor_id'][1:]:
                    tmp_proc = Processor()
                    temp_processor_list.append(tmp_proc)
                self._parse_data(label, tmp_proc, value)

        # needed to avoid adding twice cpus with same core_id (Hyperthreading/SMT)
        clock_dict: Dict[int, Dict[int, bool]] = {}

        for tmp_proc in temp_processor_list:
            # get final variable
            processor = system_info.cpu_info.get_processor(tmp_proc.get_selected_processor())
            clock = tmp_proc.core_speed
            if clock is not None:
                clock_package = clock_dict.get(tmp_proc.physical_package_id)
                if clock_package is None:
                    clock_package = {}
                    clock_dict[tmp_proc.physical_package_id] = clock_package
                if clock_package.get(tmp_proc.core_id) is None:
                    clock_package[tmp_proc.core_id] = True
                    item = MonitoredItem(
                        str(tmp_proc.core_id),
                        f"Core #{tmp_proc.core_id}",
                        round(tmp_proc.core_speed),
                        FeatureType.CLOCK
                    )
                    system_info.cpu_info.set_clock_monitored_item(tmp_proc.physical_package_id, item)
            for attr, value in tmp_proc.__dict__.items():
                if value is not None:
                    processor.__setattr__(attr, value)

        return system_info

    def _parse_data(self, label: str, temp_processor: Processor, value: str) -> None:
        for name, labels in self.cpu_info_mapping.items():
            if label in labels[1:] and value:
                if name == 'core_speed':
                    temp_processor.core_speed = float(value) * 1000 * 1000
                elif name == 'flags':
                    temp_processor.flags = sorted(value.strip().split())
                elif name == 'bugs':
                    temp_processor.bugs = sorted(value.strip().split())
                else:
                    if name == 'specification':
                        temp_processor.name = self.clean_cpu_string(value)
                    value = labels[0](value)
                    temp_processor.__setattr__(name, value)

    @staticmethod
    def clean_cpu_string(specification: str) -> str:
        return ' '.join(re.sub(CLEAN_CPU_STRING_REGEX, '', specification, flags=re.IGNORECASE).split())


import numpy
import theano
from lasagne.init import Constant, GlorotUniform
from lasagne.layers import InputLayer, Conv2DLayer, ConcatLayer, DenseLayer, get_all_params, get_output, flatten
from lasagne.nonlinearities import rectify
from lasagne.objectives import squared_error
from lasagne.updates import rmsprop
from theano import tensor

def _create_network(available_actions_num, input_shape, visual_input_var, n_variables, variables_input_var):

    dqn = InputLayer(shape=[None, input_shape.frames, input_shape.y, input_shape.x], input_var=visual_input_var)

    dqn = Conv2DLayer(dqn, num_filters=32, filter_size=[8, 8], stride=[4, 4],
                      nonlinearity=rectify, W=GlorotUniform("relu"),
                      b=Constant(.1))
    dqn = Conv2DLayer(dqn, num_filters=64, filter_size=[4, 4], stride=[2, 2],
                      nonlinearity=rectify, W=GlorotUniform("relu"),
                      b=Constant(.1))

    dqn = Conv2DLayer(dqn, num_filters=64, filter_size=[3, 3],
                      nonlinearity=rectify, W=GlorotUniform("relu"),
                      b=Constant(.1))
    if n_variables > 0:
        variables_layer = InputLayer(shape=[None, n_variables], input_var=variables_input_var)
        dqn = ConcatLayer((flatten(dqn), variables_layer))
    dqn = DenseLayer(dqn, num_units=512, nonlinearity=rectify, W=GlorotUniform("relu"), b=Constant(.1))

    dqn = DenseLayer(dqn, num_units=available_actions_num, nonlinearity=None)
    return dqn

def create_dqn(available_actions_num, input_shape, n_variables, learning_rate=0.00025, discount_factor=0.99):
    # Creates the input variables
    state = tensor.tensor4("State")
    state_after_action = tensor.tensor4("Next state")
    variables = tensor.matrix("Variables")
    variables_after_action = tensor.matrix("Next variables")
    action = tensor.vector("Actions", dtype="int32")
    reward = tensor.vector("Rewards")
    nonterminal = tensor.vector("Nonterminal", dtype="int8")

    network = _create_network(available_actions_num, input_shape, state, n_variables, variables)
    target_network = _create_network(available_actions_num, input_shape, state_after_action, n_variables,
                                     variables_after_action)
    q_values = get_output(network)
    next_q_values = get_output(target_network)
    target_action_q_value = tensor.clip(
        reward + discount_factor * nonterminal * tensor.max(next_q_values, axis=1, keepdims=False), -1, 1)
    target_q_values = tensor.set_subtensor(q_values[tensor.arange(q_values.shape[0]), action], target_action_q_value)
    loss = squared_error(q_values, target_q_values).mean()

    params = get_all_params(network, trainable=True)
    updates = rmsprop(loss, params, learning_rate)

    function_learn = theano.function([state, variables, action, state_after_action, variables_after_action, reward,
                                      nonterminal], loss, updates=updates, name="learn_fn", on_unused_input="ignore")
    function_get_q_values = theano.function([state, variables], q_values, name="eval_fn", on_unused_input="ignore")
    function_get_best_action = theano.function([state, variables], tensor.argmax(q_values), name="best_action_fn",
                                               on_unused_input="ignore")
    function_get_max_q_value = theano.function([state, variables], tensor.max(q_values, axis=1), name="max_q_fn",
                                               on_unused_input="ignore")
    return (network, target_network, function_learn, function_get_q_values, function_get_best_action,
            function_get_max_q_value)


#! /usr/bin/env python3

"""plot figures for mirna editing events"""

import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
from barplot import X_COL, Y_COL

EDITED_IN_COL = "EDITED IN"

def count_seed_editing_events(df: pd.DataFrame) -> pd.DataFrame:
    if df.shape[0] < 1:
        return pd.DataFrame({})

    seed_edit = set()
    for _, mirline in df[df.TYPE == "miRNA"].iterrows():
        seq_len = len(mirline.REF_SEQ)
        edit_pos = mirline.REF_SEQ_POS + 1

        pass_strand_seed_pos = (
            mirline.STRAND == "-" and edit_pos < seq_len and edit_pos > seq_len - 8
        )
        prim_strand_seed_pos = mirline.STRAND == "+" and edit_pos > 1 and edit_pos < 8

        if pass_strand_seed_pos or prim_strand_seed_pos:
            seed_edit.add(mirline.POS)

    df[EDITED_IN_COL] = "outside seed region"
    for mir_pos in seed_edit:
        df[EDITED_IN_COL] = df.apply(
            in_seed_region(mir_pos),
            axis=1,
        )

    df[X_COL] = df.apply(lambda row: row.REF + " -> " + row.ALT, axis=1)
    return df

def in_seed_region(pos: int) -> str:
    def func(row: pd.Series) -> str:
        return (
            "seed region" if pos > row.START and pos < row.END else row[EDITED_IN_COL]
        )

    return func

def plot_seed_editing_events_count(df: pd.DataFrame, name: str, result_image: str):
    data = (
        pd.crosstab(df[EDITED_IN_COL], df[X_COL])
        .melt(ignore_index=False, value_name=Y_COL)
        .reset_index()
    )

    sns.set_context("poster")
    fig, ax = plt.subplots(figsize=(20, 14))
    fig.suptitle(f"Editing events in seed regions and outside seed regions in\n{name}")

    palette = dict(
        zip(
            ["seed region", "outside seed region"],
            sns.color_palette(n_colors=2),
        )
    )
    sns.barplot(
        data=data.sort_values(Y_COL),
        x=X_COL,
        y=Y_COL,
        hue=EDITED_IN_COL,
        ax=ax,
        palette=palette,
    )
    fig.savefig(result_image)


import os
import json
import glob

# Windows specific Imports
from platform import system
if system() == 'Windows':
    import pathlib
    temp = pathlib.PosixPath
    pathlib.PosixPath = pathlib.WindowsPath

BASE_DIR = os.path.dirname(os.path.realpath(__file__))
UPLOAD = os.path.join(BASE_DIR, 'upload')
MODELS = os.path.join(BASE_DIR, 'static', 'models')

NO_MASK = 0
INCORRECT = 1
MASK = 2

def box_resp(mylist):
    json_resp = []
    for i in range(len(mylist)):
        (x0,y0,x1,y1) = mylist[i][0]
        prediction = mylist[i][1]
        json_resp.append({
                'top': str(y0),
                'bottom': str(y1),
                'left': str(x0),
                'right': str(x1),
                'result': prediction,
        })
    return json.dumps(json_resp)

def delete_uploads():
    """Deletes all files in UPLOAD folder """
    files = glob.glob(os.path.join(UPLOAD, '*.png'), recursive=True)
    for f in files:
        try:
            os.remove(f)
        except OSError as e:
            print("Error: %s : %s" % (f, e.strerror))
    return "All Good!"


from collections import defaultdict

class Solution:
    def accountsMerge(self, accounts: List[List[str]]) -> List[List[str]]:

        email_to_ids = defaultdict(set)
        for i, v in enumerate(accounts):
            for email in v[1:]:
                email_to_ids[email].add(i)

        visited = [False for _ in accounts]
        ret = []
        for i, v in enumerate(accounts):
            if not visited[i]:
                emails = set()
                self.dfs(i, accounts, email_to_ids, emails, visited)
                ret.append([v[0]] + sorted(emails))

        return ret

    def dfs(self, i, accounts, email_to_ids, emails, visited):
        visited[i] = True
        for email in accounts[i][1:]:
            emails.add(email)
            for nbr in email_to_ids[email]:
                if not visited[nbr]:
                    self.dfs(nbr, accounts, email_to_ids, emails, visited)

    def accountsMerge_error(self, accounts: List[List[str]]) -> List[List[str]]:

        email_id = {}
        id_emails = defaultdict(list)
        for i in range(len(accounts)):
            person = None
            for email in accounts[i][1:]:
                if email in email_id:
                    person = email_id[email]
                    break

            for email in accounts[i][1:]:
                if person is None:
                    person = i
                    email_id[email] = person
                    id_emails[person].append(email)
                elif email not in email_id:
                    email_id[email] = person
                    id_emails[person].append(email)

        ret = []
        for k, v in id_emails.items():
            ret.append([accounts[k][0]] + sorted(v))

        return ret


import enum
import html
from typing import Any, List, Optional, Sequence, Tuple

from google.protobuf import empty_pb2
from google.protobuf import text_format
from google.rpc import code_pb2
from intrinsic.kubernetes.workcell_spec.proto import installer_pb2
from intrinsic.kubernetes.workcell_spec.proto import installer_pb2_grpc
from intrinsic.logging.proto import log_item_pb2
from intrinsic.perception.proto import frame_pb2
from intrinsic.solutions import camera_utils
from intrinsic.solutions import errors as solutions_errors
from intrinsic.solutions import ipython
from intrinsic.solutions import structured_logging

ERROR_SEPARATOR = '\n\n========\n'
NO_ERROR_FOUND_MSG = 'No error data found.'
_ERROR_REPORT_EVENT_SOURCE = 'error_report'
_COLLAPSIBLE_ERROR_HEADER_HTML = """
<meta name="viewport" content="width=device-width, initial-scale=1">

"""
_ERROR_INTRO_HTML = """
<legend style="color:#900; font-weight: bold">Errors summary:</legend>"""
_COLLAPSIBLE_SCRIPT_HTML = """"""

def _error_recovery_instructions_list_html(err: 'ErrorInstance') -> str:
  """Converts all recovery instructions into a human readable html list.

  Args:
    err: Error to retrieve instructions from.

  Returns:
    html string representation of error recovery instructions
  """
  if not err.recovery_instructions:
    return ''

  recovery_instructions = '
'
  for instruction in err.recovery_instructions:
    recovery_instructions += f'
{instruction.human_readable}
'
  recovery_instructions += '
'

  images = ''

  for d in err.data:
    if d.data and d.data.type_url:
      if frame_pb2.Frame.DESCRIPTOR.full_name in d.data.type_url:
        frame = camera_utils.get_frame_from_any(d.data)
        img = camera_utils.get_encoded_frame(frame)
        images += f""

  return (
      '
'
      '  Recovery Instructions:'
      f'     {recovery_instructions}'
      f'     {images}'
  )

def _error_to_collapsible_button_html(
    err: 'ErrorInstance',
    internal_errors: Optional[List['ErrorInstance']] = None,
) -> str:
  """Prepares html for an interactive button for an error.

  Args:
    err: Error summary
    internal_errors: Optional. Additional errors nested under this error.

  Returns:
    html representation for interactive error button.
  """
  summary = html.escape(err.human_readable_summary.replace('\n', '
'))
  error_message = html.escape(err.error_message)

  # Internal errors are represented as buttons that are only visible when the
  # parent error is uncollapsed.
  internal_buttons_html = ''
  if internal_errors:
    internal_buttons = [
        _error_to_collapsible_button_html(e) for e in internal_errors
    ]
    internal_buttons_html = ''.join(internal_buttons)

  recovery_instructions = _error_recovery_instructions_list_html(err)

  return (
      '
'
      f'  {summary}'
      '
'
      '

'
      f'
{error_message}
'
      f'  {recovery_instructions}'
      f'  {internal_buttons_html}'
      '

'
      '
'
  )

class PrintLevel(enum.Enum):
  """Config for how much detail is printed about errors."""

  OFF = 1  # No printing done.
  SUMMARY = 2  # Only a short summary about the error is printed.
  FULL_ERROR_REPORT = 3  # The full proto ErrorReport is printed.

class ErrorInstance:
  """ErrorReport augmented by further insights from error processing.

  Attributes: log_item_proto
    uid: LogItem unique ID of error.
    parent_error_uid: Set if this error happened in the scope of an error of a
      higher-level component. The UID is the LogItem UID of the parent.
  """

  def __init__(self, log_item: log_item_pb2.LogItem):
    """Constructs an ErrorInstance object.

    Args:
      log_item: LogItem proto for this ErrorInstance.
    """
    self._log_item_proto: log_item_pb2.LogItem = log_item

    # Set if this error happened in the scope of an error of a higher-level
    # component. The UID is the LogItem UID of the parent.
    self.parent_error_uid: Optional[int] = None

  def __str__(self) -> str:
    return f"{self.summary}\n\n For more details use the 'details' property."

  @property
  def uid(self) -> int:
    """LogItem unique ID of error."""
    return self._log_item_proto.metadata.uid

  @property
  def log_item_proto(self) -> log_item_pb2.LogItem:
    return self._log_item_proto

  @property
  def summary(self) -> str:
    """String representing a summary view on the error summary.

    Returns:
      String representing the summary
    """
    return (
        f'Error: {self.human_readable_summary} \nError Status:  '
        f' {code_pb2.Code.Name(self.log_item_proto.payload.error_report.description.status.code)}\n'
        f'  Message: {self.error_message}'
    )

  @property
  def details(self) -> str:
    """String representing a detailed view on the error summary.

    Returns:
      String representing the details
    """
    return (
        f'{self.summary}\n\nFull LogItem:\n'
        f'{text_format.MessageToString(self.log_item_proto)}'
    )

  @property
  def human_readable_summary(self) -> str:
    """String representing the human readable summary of the error.

    Returns:
      String representing the human readable summary
    """
    return (
        self.log_item_proto.payload.error_report.description.human_readable_summary
    )

  @property
  def error_message(self) -> str:
    """String representing the error message.

    Returns:
      String representing the error message
    """
    return self.log_item_proto.payload.error_report.description.status.message

  @property
  def recovery_instructions(self) -> Sequence[Any]:
    """Retrieves list of recovery instructions for this error.

    Returns:
      List of recovery instructions
    """
    return self.log_item_proto.payload.error_report.instructions.items

  @property
  def data(self) -> Sequence[Any]:
    """Retrieves list of additional data for this error.

    Returns:
      List of additional data
    """
    return self.log_item_proto.payload.error_report.data.items

  def additional_information(self, e: 'ErrorInstance') -> bool:
    """Checks whether additional information is provided by given error.

    Args:
      e: Error to compare for information

    Returns:
      False if given error does not provide additional Information
    """
    if self == e:
      return True
    if self.error_message == e.error_message:
      if self.recovery_instructions and not e.recovery_instructions:
        return False
      if self.recovery_instructions == e.recovery_instructions:
        if self.data and not e.data or self.data == e.data:
          return False
    return True

class ErrorGroup:
  """List of error summaries with utilities.

  Attributes: errors
    workcell_health_issue: Empty if the workcell was healthy when this object
      was constructed, otherwise a description of the health status.
  """

  def __init__(
      self, errors: List[ErrorInstance], workcell_health_issue: str = ''
  ):
    """Constructs an ErrorGroup object.

    Args:
      errors: List of ErrorInstance objects.
      workcell_health_issue: Empty if the workcell is healthy when this object
        is constructed, otherwise a description of the health status.
    """
    self._errors: List[ErrorInstance] = errors

    # We reverse to order to show the most recent error first.
    def negative_acquisition_time(e: ErrorInstance) -> Tuple[int, int]:
      return (
          -e.log_item_proto.metadata.acquisition_time.seconds,
          -e.log_item_proto.metadata.acquisition_time.nanos,
      )

    self._errors.sort(key=negative_acquisition_time)

    self._process_error_hierarchy()
    self.workcell_health_issue: str = workcell_health_issue

  def __str__(self) -> str:
    return self.summary

  @property
  def errors(self) -> List[ErrorInstance]:
    return self._errors

  @property
  def summary(self) -> str:
    """String representing a summary view on the error summaries.

    Returns:
      String representing the summary view
    """
    result = ''
    if self.workcell_health_issue:
      result += self.workcell_health_issue
    if not self.errors:
      result += NO_ERROR_FOUND_MSG
    else:
      result += '\n===Errors summary:===\n'
      result += ERROR_SEPARATOR.join([e.summary for e in self.errors])
    return result

  @property
  def details(self) -> str:
    """String representing a detailed view on the error summaries.

    Returns:
      String representing the details including the full ErrorReport proto.
    """
    result = ''
    if self.workcell_health_issue:
      result += self.workcell_health_issue
    if not self.errors:
      result += NO_ERROR_FOUND_MSG
    else:
      result += '\nDetailed error summaries:\n'
      result += ERROR_SEPARATOR.join([e.details for e in self.errors])
    return result

  def print_info(self, print_level: PrintLevel = PrintLevel.SUMMARY) -> None:
    """Print information about errors.

    Args:
      print_level: Config for how much detail is printed.
    """
    if print_level is PrintLevel.OFF:
      return
    elif print_level is PrintLevel.SUMMARY:
      print(self.summary)
    elif print_level is PrintLevel.FULL_ERROR_REPORT:
      print(self.details)
    else:
      raise solutions_errors.InvalidArgumentError(
          f'Unknown PrintLevel: {print_level} (value = {print_level.value})'
      )

  def display_only_in_ipython(self) -> None:
    """Display an object but only if running in IPython."""
    ipython.display_html_if_ipython(self._repr_html_(), newline_after_html=True)

  def _process_error_hierarchy(self) -> None:
    """Extracts hierarchical relationships between instances in self.errors.

    The results are stores within the error instances.
    """
    for err in self.errors:
      # Process subskill invocations. A subskill is expected to contain the ID
      # of its parent skill error in its LogItem context.
      if err.log_item_proto.context.parent_skill_id != 0:
        # Note that we currently assume only one nesting level.
        def is_parent(
            parent: ErrorInstance, child: ErrorInstance = err
        ) -> bool:
          """Evaluates whether 'child' is a child of 'parent'."""
          return parent.log_item_proto.context.parent_skill_id == 0 and (
              parent.log_item_proto.context.skill_id
              == child.log_item_proto.context.parent_skill_id
          )

        potential_parents = list(filter(is_parent, self.errors))
        if len(potential_parents) == 1:
          err.parent_error_uid = potential_parents[0].uid
        else:
          print(
              'Warning: Unexpected parent relationship of the error with UID'
              f' {err.uid}. Expected 1 parent but found '
              f'{len(potential_parents)}.'
          )

  def _get_top_level_errors(self) -> List[ErrorInstance]:
    """Returns all errors which are not nested below another parent error.

    Returns:
      List of errors which do not have child errors.
    """
    return list(filter(lambda err: not err.parent_error_uid, self.errors))

  def _get_children(self, uid: int) -> List[ErrorInstance]:
    """Returns all errors that are nested below a given error.

    Args:
      uid: LogItem UID of error.

    Returns:
      List children.
    """
    return list(
        filter(
            lambda e: e.parent_error_uid and e.parent_error_uid == uid,
            self.errors,
        )
    )

  def _get_unique_top_level_errors(self) -> List[ErrorInstance]:
    """Returns all errors which are not nested below another parent error.

    Additionally removes errors which group other top-level errors.

    Returns:
      List of unique errors which do not have child errors.
    """
    top_level_errors = self._get_top_level_errors()

    keep = []
    for e in top_level_errors:
      keep.append(e)
      for x in top_level_errors:
        if x not in keep:
          break
        if not e.additional_information(x):
          keep.remove(x)
        elif not x.additional_information(e):
          keep.remove(e)
          break

    return keep

  def _repr_html_(self) -> str:
    """Used to display rich information about errors in a Jupyter notebook.

    In a Jupyter notebook display(<this_object>) will render this HTML.

    Returns:
      Rich html representation of errors.
    """

    errors_repr = [
        _error_to_collapsible_button_html(
            err, internal_errors=self._get_children(err.uid)
        )
        for err in self._get_unique_top_level_errors()
    ]

    body = ''
    if self.errors:
      body = f"""
 <fieldset>
 {_ERROR_INTRO_HTML}
 {html.escape(self.workcell_health_issue)}
 {"".join(errors_repr)}
 </fieldset>
 {_COLLAPSIBLE_SCRIPT_HTML}
 """
    else:
      body = 'No errors found.'
    return f"""{_COLLAPSIBLE_ERROR_HEADER_HTML}

                  {body}
                """

class ErrorsLoader:
  """Extracts error reports and prepares for presentation to user.

  ErrorReports are structured data which is logged by various intrinsic
  components at
  runtime. This class uses the DataLogger as the backend storing the reports. It
  is both used as an internal helper class within workcell client modules to
  expose errors, as well as a separate module by the user.
  """

  def __init__(
      self,
      structured_logs: structured_logging.StructuredLogs,
      installer_service_stub: installer_pb2_grpc.InstallerServiceStub,
  ):
    """Constructs an Entity object.

    Note that already during construction, we fetch the list of children and
    cache them.

    Args:
      structured_logs: DataLogger client.
      installer_service_stub: Installer service to check the workcell health.
    """
    self._structured_logs: structured_logging.StructuredLogs = structured_logs
    self._installer_service_stub: installer_pb2_grpc.InstallerServiceStub = (
        installer_service_stub
    )

  def list_error_data(
      self,
      executive_plan_id: Optional[int] = None,
      last_seconds_to_read: int = 1200,
  ) -> ErrorGroup:
    """Load all ErrorReports for a given plan execution.

    This call might take a few seconds as data needs to be loaded from the
    backend.

    Note that currently only data for the last 6 hours is stored on prem. By
    default only the last 20 min are considered. If you have a longer execution
    or would like to load errors from executions in the past, adjust
    'last_seconds_to_read'.

    Args:
      executive_plan_id: Optional Log ID of the plan to consider. Can be e.g.
        found in ExecutiveState.
      last_seconds_to_read: Only considers recent logs within this timeframe

    Returns:
      List of LogItems containing metadata and the ErrorReport payload
    """
    print('Reading error reports...', end='')
    log_items = self._structured_logs.query(
        _ERROR_REPORT_EVENT_SOURCE, last_seconds_to_read
    )
    print('Done.')
    if not log_items:
      return ErrorGroup([], workcell_health_issue=self._load_workcell_health())
    if not executive_plan_id:
      return ErrorGroup(
          [ErrorInstance(proto) for proto in log_items],
          workcell_health_issue=self._load_workcell_health(),
      )
    log_item_proto_list = list(
        filter(
            lambda item: item.context.executive_plan_id == executive_plan_id,
            log_items,
        )
    )
    return ErrorGroup(
        [ErrorInstance(proto) for proto in log_item_proto_list],
        workcell_health_issue=self._load_workcell_health(),
    )

  def _load_workcell_health(self) -> str:
    """Loads the workcell health and returns an error message if not healthy.

    No retries, to avoid blocking the summary of errors.

    Returns:
      Empty if healthy, description otherwise.
    """
    try:
      response = self._installer_service_stub.GetInstalledSpec(
          empty_pb2.Empty()
      )
      error_reason = ''
      if isinstance(response.error_reason, str) and response.error_reason:
        error_reason = '\n\tError reason: ' + response.error_reason
      if response.status == installer_pb2.GetInstalledSpecResponse.HEALTHY:
        return ''
      elif response.status == installer_pb2.GetInstalledSpecResponse.PENDING:
        return (
            'WORKCELL NOT HEALTHY: Workcell backends not healthy now but '
            'expected to become healthy again automatically.'
            + error_reason
            + '\n'
        )
      elif response.status == installer_pb2.GetInstalledSpecResponse.ERROR:
        return (
            'WORKCELL NOT HEALTHY: Workcell backend is unhealthy and not '
            'expected to recover without intervention. Try restarting your '
            'app.'
            + error_reason
            + '\n'
        )
    except Exception:  # pylint: disable=broad-except
      return 'WORKCELL HEALTH UNKNOWN: Could not load the workcell status.\n'
    return (
        'WORKCELL HEALTH UNKNOWN: Unknown health status of workcell detected.\n'
    )


# https://atcoder.jp/contests/abc007/tasks/abc007_3

from collections import deque

class Point:
    def __init__(self, x, y, d):
        super().__init__()
        self.x = x
        self.y = y
        self.d = d

    def __repr__(self):
        return 'x {} y {} d {}'.format(self.x, self.y, self.d)

if __name__ == "__main__":
    R, C = map(int, input().split())
    sy, sx = map(int, input().split())
    gy, gx = map(int, input().split())
    sy -= 1
    sx -= 1
    gx -= 1
    gy -= 1
    A = []
    for _ in range(R):
        row = [c for c in input()]
        A.append(row)

    queue = deque()
    queue.appendleft(Point(sx, sy, 0))

    result = -1
    while queue:
        point = queue.pop()
        if point.x == gx and point.y == gy:
            result = point.d
            break

        for dx, dy in [(-1, 0), (1, 0), (0, -1), (0, 1)]:
            x = point.x + dx
            y = point.y + dy
            if A[y][x] == '.':
                A[y][x] = '#'
                queue.appendleft(Point(x, y, point.d+1))

    print(result)


import settings
from django.test import TestCase
import sys, os, vcr, dhis2
from dhis2.dhis2_match import *
from dhis2.models import Dhis2_Mtrac_Indicators_Mapping
import json
from mock import *
from eav.models import Attribute

FIXTURES = os.path.abspath(dhis2.__path__[0]) + "/tests/fixtures/cassettes/"

DHIS2_HMIS033b_INDICATOR_URL= settings.DHIS2_HMIS033b_INDICATOR_URL
DHIS2_API_URL = settings.DHIS2_BASE_URL+ '/api/'

JSON_EXTENSION = '.json'

class Test_Dhis2_Fetch_Health_Indicators(TestCase):

    def setUp(self):
        self.fetcher = Dhis2_Fetch_Health_Indicators(97)

    def test_get_hmis033b_dataset(self):
       with vcr.use_cassette(FIXTURES + self.__class__.__name__ + "/" + sys._getframe().f_code.co_name + ".yaml"):
           obj = self.fetcher.fetch(JSON_EXTENSION, DHIS2_HMIS033b_INDICATOR_URL)
           self.assertIsNotNone(obj)

    def test_clean_indicator_names_from_dhis2(self):
        self.assertEqual(self.fetcher.clean_indicator_names_from_dhis2('Bubukwanga - WEP    '), 'Bubukwanga')
        self.assertEqual(self.fetcher.clean_indicator_names_from_dhis2('Bubukwanga (helloo there , remove me)'), 'Bubukwanga')
        self.assertEqual(self.fetcher.clean_indicator_names_from_dhis2('Bubukwanga - WEP  (asdasdasd ) '), 'Bubukwanga')
        self.assertEqual(self.fetcher.clean_indicator_names_from_dhis2('Bubukwanga  (asdasdasd) - WEP '), 'Bubukwanga')
        self.assertEqual(self.fetcher.clean_indicator_names_from_dhis2('(asdasdasd) - WEP '), 'asdasdasd')
        self.assertEqual(self.fetcher.clean_indicator_names_from_dhis2(' (asdasdasd) '), 'asdasdasd')

    def test_compare_strings(self):
        self.assertEqual(self.fetcher.compare_strings('Bubukwanga', 'Bubukwanga'), 100)
        self.assertEqual(self.fetcher.compare_strings('Bubukwanga', 'Bundibugyo'), 50)
        self.assertEqual(self.fetcher.compare_strings('a', 'b'), 0)

    def test_get_indicators_names_match_level(self):
        self.assertTrue(self.fetcher.get_indicators_names_match_level("Adverse Events Following Immunization Cases - WEP","Adverse Events Following Immunization Cases")==100)
        self.assertTrue(self.fetcher.get_indicators_names_match_level("some name - WEP","soome name")>90)

    def test_find_matching_indicator_from_mtrack(self):
        test_dhis2_name  = u'Adverse Events Following Immunization Cases - WEP'
        expected = u'Adverse Events Following Immunization cases'
        matching_mtrack= self.fetcher.find_matching_indicator_from_mtrack(test_dhis2_name)
        self.assertEquals(matching_mtrack.id , 360)

    def test_find_matching_known_indicator_from_mtrack(self):
        test_dhis2_name  = u'(Yellow Used)'
        matching_mtrack= self.fetcher.find_matching_indicator_from_mtrack(test_dhis2_name)
        self.assertEquals(matching_mtrack.id , 340)

    def test_find_matching_indicator_from_mtrack(self):
        test_dhis2_name  = u'I DONT EXIST IN MTRACK...YEAHHHH - WEP'
        self.assertEquals(self.fetcher.find_matching_indicator_from_mtrack(test_dhis2_name),None)

    def test_find_matches_and_update_mapping_table(self):
        Dhis2_Mtrac_Indicators_Mapping.objects.all().delete()
        arbitrary_disease = {"name":u"Typhoid Fever Cases - WEP", "id": u"IkI01xB7RIi", "href":DHIS2_API_URL + "dataElements/IkI01xB7RIi", "combo_id":u"92DkrSOchnL" }
        self.fetcher.find_matches_and_update_mapping_table(arbitrary_disease)
        record = Dhis2_Mtrac_Indicators_Mapping.objects.filter(dhis2_uuid='IkI01xB7RIi')
        self.assertIsNotNone(record)
        self.assertEquals(len(record), 1)
        record = record[0]
        self.assertEquals( record.dhis2_uuid, arbitrary_disease['id'])
        self.assertEquals( record.dhis2_combo_id, arbitrary_disease['combo_id'])
        self.assertEquals( record.eav_attribute, Attribute.objects.get(slug='cases_tf'))

    def test_malaria_case_is_mapped_to_cases_ma_slug_and_NOT_doc_ma(self):
        Dhis2_Mtrac_Indicators_Mapping.objects.all().delete()
        disease = {"name":u"Malaria Cases - WEP", "id": u"fclvwNhzu7d", "href":DHIS2_API_URL + "dataElements/fclvwNhzu7d", "combo_id":u"92DkrSOchnL" }
        self.fetcher.find_matches_and_update_mapping_table(disease)
        record = Dhis2_Mtrac_Indicators_Mapping.objects.filter(dhis2_uuid='fclvwNhzu7d')
        self.assertIsNotNone(record)
        self.assertEquals(len(record), 1)
        record = record[0]
        self.assertEquals( record.dhis2_uuid, disease['id'])
        self.assertEquals( record.dhis2_combo_id, disease['combo_id'])
        self.assertEquals( record.eav_attribute, Attribute.objects.get(slug='cases_ma'))
        self.assertNotEquals(record.eav_attribute, Attribute.objects.get(slug='doc_ma'))

    def test_get_indicator_combo_option_id_default(self):
        category_combo_url = DHIS2_API_URL + 'categoryCombos/92DkrSOchnL'
        with vcr.use_cassette(FIXTURES + self.__class__.__name__ + "/" + sys._getframe().f_code.co_name + ".yaml"):
            self.assertEqual(self.fetcher.get_indicator_combo_option_id(category_combo_url), 'gGhClrV5odI')

    def test_get_indicator_combo_option_non_default(self):
        category_combo_url = DHIS2_API_URL + 'categoryCombos/7O1Zh4rJoIn'
        with vcr.use_cassette(FIXTURES + self.__class__.__name__ + "/" + sys._getframe().f_code.co_name + ".yaml"):
            self.assertIsNone(self.fetcher.get_indicator_combo_option_id(category_combo_url) )

    def test_get_category_combos_from_combo_category_option(self):
        category_combo_url = DHIS2_API_URL + 'categoryCombos/7O1Zh4rJoIn'
        with vcr.use_cassette(FIXTURES + self.__class__.__name__ + "/" + sys._getframe().f_code.co_name + ".yaml"):
            indicators = self.fetcher.get_category_combos_from_combo_category_option(category_combo_url)

            self.assertEquals(len(indicators),12)
            for indicator in indicators :
                 self.assertIsNotNone(indicator['combo_id'])
                 self.assertIsNotNone(indicator['name'])
                 self.assertIsNotNone(indicator['href'])

    def test_get_combo_id_from_indicator(self):
        url = DHIS2_API_URL + 'dataElements/fTwT8uX9Uto'
        with vcr.use_cassette(FIXTURES + self.__class__.__name__ + "/" + sys._getframe().f_code.co_name + ".yaml"):
            combo_id = self.fetcher.get_combo_id_from_indicator(url)
            self.assertEquals(combo_id,'gGhClrV5odI')

    def test_update_mappings_table_with_default_indicator(self):
        element_id = 'fTwT8uX9Uto'
        disease_url = DHIS2_API_URL + 'dataElements/' + element_id
        with vcr.use_cassette(FIXTURES + self.__class__.__name__ + "/" + sys._getframe().f_code.co_name + ".yaml"):
            self.fetcher.update_mappings_table(disease_url)
            record = Dhis2_Mtrac_Indicators_Mapping.objects.filter(dhis2_uuid= element_id )
            self.assertEqual(len(record), 1)
            record=record[0]
            self.assertEqual(record.dhis2_combo_id, 'gGhClrV5odI')
            self.assertEqual(record.eav_attribute, Attribute.objects.get(slug='cases_ae'))

    def test_update_mappings_table_with_non_default_indicator(self):
        indicator_url = DHIS2_API_URL + 'dataElements/KPmTI3TGwZw'
        with vcr.use_cassette(FIXTURES + self.__class__.__name__ + "/" + sys._getframe().f_code.co_name + ".yaml"):
            self.fetcher.update_mappings_table(indicator_url)
            record = Dhis2_Mtrac_Indicators_Mapping.objects.filter(dhis2_uuid= 'KPmTI3TGwZw')
            self.assertEqual(len(record), 7)

            record_mtrac = sorted([a_record.eav_attribute.id for a_record in record])
            expected_list = sorted([Attribute.objects.get(slug='test_sm').id,
                             Attribute.objects.get(slug='test_rdt').id,
                             Attribute.objects.get(slug='test_rdp').id,
                             Attribute.objects.get(slug='test_mtc').id,
                             Attribute.objects.get(slug='test_mtp').id,
                             Attribute.objects.get(slug='test_pcc').id,
                             Attribute.objects.get(slug='test_pcy').id ])

            self.assertListEqual(record_mtrac , expected_list)

    def test_fetch_elements_from_dataset(self):
        dataset_url = DHIS2_API_URL + 'dataSets/V1kJRs8CtW4'
        with vcr.use_cassette(FIXTURES + self.__class__.__name__ + "/" + sys._getframe().f_code.co_name + ".yaml"):
            elements = self.fetcher.fetch_elements_for_dataset(dataset_url)
            self.assertTrue(len(elements),39)


from django.shortcuts import render
from django.http import HttpResponse
import requests
import os
import json
from bs4 import BeautifulSoup
from .models import Greeting
from .skr import getdatas,maximizeRate,makeoutputDic,manycontest,getdatasa

# Create your views here.
def index(request):
    return vote(request)

def result(request,username):
    r = ""
    name = username
    sex = "male"
    return last(request,name,sex,r)

def resultr(request,username):
    r = "r"
    name = username
    sex = "female"
    return last(request,name,sex,r)

def vote(request):
    name = "hdnkt"
    sex = "male"
    try:
        name = request.POST["username"]
        sex = request.POST["sex"]
    except:
        name = "hdnkt"
        sex = "male"
    return last(request,name,sex,"")

def last(request,name,sex,r):
    error = " "

    try:
        if sex =="female":
            r = "r"
            dic=manycontest()
            tmp,ratedHis = getdatas(name,dic)
            real = tmp[len(tmp)-1]["NewRating"]
            ans,ind=maximizeRate(tmp,ratedHis)
            final = makeoutputDic(ans,ind,tmp)
        else:
            tmp,ratedHis = getdatasa(name)
            real = tmp[len(tmp)-1]["NewRating"]
            ans,ind=maximizeRate(tmp,ratedHis)
            final = makeoutputDic(ans,ind,tmp)
    except:
        name="hdnkt"
        tmp,ratedHis = getdatasa(name)
        real = tmp[len(tmp)-1]["NewRating"]
        ans,ind=maximizeRate(tmp,ratedHis)
        final = makeoutputDic(ans,ind,tmp)
        error = "入力された名前のユーザーは存在しません。"

    highest = final[len(final)-1]["NewRating"]
    dist = "+"+str(highest-real) if highest-real >= 0 else str(highest-real)

    return render(request,"ratinggraph.html",{"final":final,"name":name,"error":error,"r":r,"highest":highest,"real":real,"dist":dist})

def db(request):
    greeting = Greeting()
    greeting.save()
    greetings = Greeting.objects.all()
    return render(request, "db.html", {"greetings": greetings})


import re
import string

from keras_preprocessing.sequence import pad_sequences
from keras_preprocessing.text import Tokenizer
from nltk.corpus import stopwords

def sanitize_text(media):
    # fill empty values

    media = media.fillna('')

    # remove punctuation

    media['media_summary'] = media['media_summary'].apply(lambda x: re.sub('[^\w\s]', '', x))
    media['media_title'] = media['media_title'].apply(lambda x: re.sub('[^\w\s]', '', x))

    media['media_genre_0'] = media['media_genre_0'].apply(lambda x: re.sub('\s', '', x))
    media['media_genre_1'] = media['media_genre_1'].apply(lambda x: re.sub('\s', '', x))
    media['media_genre_2'] = media['media_genre_2'].apply(lambda x: re.sub('\s', '', x))
    media['media_genre_3'] = media['media_genre_3'].apply(lambda x: re.sub('\s', '', x))

    # remove stop words

    stops = stopwords.words('german')

    media['media_summary'] = media['media_summary'].apply(
        lambda x: ' '.join([word for word in x.lower().translate(x.maketrans('', '', string.punctuation)).split() if
                            word not in stops])
    )

    media['media_title'] = media['media_title'].apply(
        lambda x: ' '.join([word for word in x.lower().translate(x.maketrans('', '', string.punctuation)).split() if
                            word not in stops])
    )

    return media

def process_media(media):
    # remove unneeded columns
    # not sure how to process mediaReleaseDate for now - maybe use unix seconds since 1970

    media = media.drop(columns=['media_id', 'media_release_date'])

    # remove stopwords

    media = sanitize_text(media)

    # process vote_type

    if 'vote_type' in media:
        vote_type_dict = {
            'positive': 1,
            'neutral': 0,
            'negative': -1
        }

        media['vote_type'] = media['vote_type'].apply(lambda x: vote_type_dict.get(x))

    # process media_type

    media_type_dict = {
        'movie': 1,
        'tv': 2,
    }

    media['media_type'] = media['media_type'].apply(lambda x: media_type_dict.get(x))

    # fit tokenizer for genres

    max_genres = 4

    genre_tok = Tokenizer(oov_token='<UnknownGenre>')

    for i in range(4):
        genre_tok.fit_on_texts(media[f'media_genre_{i}'])

    # map genres to tokens

    genre_cols = []

    for i in range(max_genres):
        genre_cols.append(f'media_genre_{i}')

    media[genre_cols] = media[genre_cols].apply(
        lambda x: x.apply(
            lambda y: (lambda z: z[0][0] if z[0] else float(0))(genre_tok.texts_to_sequences([y]))
        )
    )

    # fit tokenizer on summaries and titles

    prosa_tokenizer = Tokenizer(oov_token='<OOV>')

    prosa_tokenizer.fit_on_texts(media['media_summary'])
    prosa_tokenizer.fit_on_texts(media['media_title'])

    # map prosa to tokens

    summary_word_count = 60

    summary_seqs = pad_sequences(prosa_tokenizer.texts_to_sequences(media['media_summary']), maxlen=summary_word_count,
                                 padding='post')

    for i in range(summary_word_count):
        media[f'media_summary_{i}'] = summary_seqs[:, i]

    title_word_count = 5

    title_seqs = pad_sequences(prosa_tokenizer.texts_to_sequences(media['media_title']), maxlen=title_word_count,
                               padding='post')

    for i in range(title_word_count):
        media[f'media_title_{i}'] = title_seqs[:, i]

    return media.drop(columns=['media_summary', 'media_title'])


"""
Util functions for MLBootcamp
"""
import os
import re
import time
import glob
import shutil
import zipfile
import urllib.request
import pandas as pd

non_decimal = re.compile(r'[^\d.]+') # regex for removing non-decimal characters from string
def parse_beds(string):
    strings = string.lower().split(', ')
    num_beds = None
    for s in strings:
        if "bd" in s:
            try:
                num_beds = float(non_decimal.sub('', s))
            except Exception as e:
                pass
        # treat studio as 0 bedrooms
        elif "studio" in s.lower():
            num_beds = 0
        return num_beds

def parse_bath(string):
    strings = string.lower().split(', ')
    num_bath = None
    for s in strings:
        if "ba" in s:
            try:
                num_bath = float(non_decimal.sub('', s))
            except Exception as e:
                pass
            finally:
                return num_bath
def parse_sqft(string):
    strings = string.lower().split(', ')
    sqft = None
    for s in strings:
        if "ft" in s:
            try:
                sqft = float(non_decimal.sub('', s))
            except Exception as e:
                pass
            finally:
                return sqft

def parse_property_type(string):
    # property types mapping
    property_types = {'Condo For Sale': 'condo',
                      'House For Sale': 'house',
                      'Apartment For Sale': 'apartment',
                      'New Construction': 'new',
                      'Foreclosure': 'foreclosure',
                       'Lot/Land For Sale': 'lot',
                      'Coming Soon': 'coming',
                      'Co-op For Sale': 'coop',
                      'Auction': 'auction',
                      'For Sale by Owner': None,
                      'Townhouse For Sale': 'townhouse'}
    try:
        property_type = property_types[string]
    except KeyError as e:
        print(e)
        property_type = None
    finally:
        return property_type

def download_extract_data(data_dir: str):
    """
    Download zipfile and extract its contents to data_dir
    Args:
        data_dir: extract data here
    """
    if not os.path.exists(data_dir):
        os.makedirs(data_dir)
    data_save_path = os.path.join(data_dir, 'data.zip')
    data_download_url = 'https://github.com/kylehounslow/datasets/raw/master/mlbootcamp/sf_housing.zip'
    print('downloading data...')
    urllib.request.urlretrieve(data_download_url, data_save_path)
    print('extracting data to {data_dir}...'.format(data_dir=data_dir))
    data_zip = zipfile.ZipFile(data_save_path)
    data_zip.extractall(data_dir)
    data_zip.close()
    print('Done.')

def load_data_to_dataframe(data_dir: str) -> pd.DataFrame:
    """
    Load all .csv files from data_dir and concatenate them into a single DataFrame.
    Args:
        data_dir: path to data directory
    Returns:
        pd.DataFrame: all data from files in data_dir
    Notes:
        All duplicate rows will be discarded
    """
    all_csvs = []
    # load the csv files from all scraping runs
    csv_filenames = os.path.join(data_dir, '**/*.csv')
    print('loading data {csv_filenames}'.format(csv_filenames=csv_filenames))
    for filename in glob.glob(csv_filenames):
        all_csvs.append(pd.read_csv(filename))
    # combine all dataframes together and drop any duplicate entries
    df = pd.concat(all_csvs, ignore_index=True).drop_duplicates()
    print("Found a total of {count} data points".format(count=len(df)))
    # save this combined dataframe as csv for safe keeping
    df.to_csv(os.path.join(data_dir, 'all_data.csv'), index=False)
    return df

def format_price(price: str):
    """
    Remove all non-numerical values from price column
    Args:
        price: string representation of price
    Returns:
        price_num: floating point representation of price
    """
    price = str(price)
    multiply_factor = 1
    if 'M' in price:
        multiply_factor = 1e6
    elif 'K' in price:
        multiply_factor = 1e3
    non_decimal = re.compile(r'[^0-9\.]')
    price_num = None
    try:
        price_num = float(non_decimal.sub('', price))*multiply_factor
    except Exception as e:
#         print(f'error converting \"{price}\": {e}')
        pass
    finally:
        return price_num
def load_sf_housing_dataset():
    """
    Download, extract, and format the housing dataset.
    Returns:
        pd.DataFrame: formatted dataset
    """
    data_dir = './tmp'
    download_extract_data(data_dir=data_dir)
    df = load_data_to_dataframe(data_dir=data_dir)
    df['price'] = df.price.apply(format_price)
    df['bed'] = df['facts and features'].apply(parse_beds)
    df['bath'] = df['facts and features'].apply(parse_bath)
    df['sqft'] = df['facts and features'].apply(parse_sqft)
    df['property_type'] = df['title'].apply(parse_property_type)
    if os.path.exists(data_dir):
        shutil.rmtree(data_dir)
    return df


from PyQt5 import QtCore
from PyQt5.QtGui import QColor
from PyQt5.QtWidgets import *

## ==> SPLASH SCREEN
from ui_SplashScreen import Ui_SplashScreen

## ===> Design's Functions
from ui_functions import UIFunctions

counter = 0

# SPLASH SCREEN
class SplashScreen(QMainWindow):
    def __init__(self):
        QMainWindow.__init__(self)
        self.ui = Ui_SplashScreen()
        self.setGeometry(400, 150, 0, 0)
        self.ui.setupUi(self)

        self.functions = UIFunctions(self)

        ## UI ==> INTERFACE CODES
        ########################################################################

        ## REMOVE TITLE BAR
        self.functions.removeTileBare()

        ## DROP SHADOW EFFECT
        self.shadow = QGraphicsDropShadowEffect(self)
        self.shadow.setBlurRadius(20)
        self.shadow.setXOffset(0)
        self.shadow.setYOffset(0)
        self.shadow.setColor(QColor(0, 0, 0, 60))
        self.ui.dropShadowFrame.setGraphicsEffect(self.shadow)

        ## QTIMER ==> START
        self.timer = QtCore.QTimer()
        self.timer.timeout.connect(self.progress)
        # TIMER IN MILLISECONDS
        self.timer.start(35)

        ## SHOW ==> MAIN WINDOW
        ########################################################################
        self.show()
        ## ==> END ##


import torch
import torch.nn as nn
import torch.nn.functional as F
from .GCN_layer import GCN as GraphConv_ordi

class ConvGCN(nn.Module):
    def __init__(self, in_features, num_class, hidden2):
        super(ConvGCN, self).__init__()

        # classification
        self.hidden2 = hidden2
        self.convs2 = nn.ModuleList()
        self.batch_norms2 = nn.ModuleList()
        for i in range(len(self.hidden2)):
            if i == 0:
                conv = GraphConv_ordi(in_features=in_features, out_features=self.hidden2[0])
            else:
                conv = GraphConv_ordi(in_features=self.hidden2[i - 1], out_features=self.hidden2[i])
            self.convs2.append(conv)
            self.batch_norms2.append(nn.BatchNorm1d(self.hidden2[i]))

        self.mlp = nn.Sequential(
            nn.Linear(sum(self.hidden2) * 2, self.hidden2[-1] * 4),
            # nn.BatchNorm1d(self.hidden2[-1]),
            nn.ReLU(True),
            nn.Linear(self.hidden2[-1] * 4, num_class)
        )

    def forward(self, x, edge):
        xs = []
        for conv, bn in zip(self.convs2, self.batch_norms2):
            x = conv(x, edge)
            x = F.relu(bn(x.transpose(1, 2)).transpose(1, 2))
            xs.append(x)
        graph = torch.cat([self.readout(xx) for xx in xs], dim=-1)  # [B,P,d*xx]
        out = self.mlp(graph)
        return out

    def readout(self, x):
        return torch.concat([global_max_pool(x), global_mean_pool(x)], dim=-1)

def global_max_pool(x):  # x:[B,P,d]
    return F.max_pool1d(x.transpose(1, 2), x.shape[1]).squeeze(2)

def global_mean_pool(x):  # x:[B,P,d]
    return F.avg_pool1d(x.transpose(1, 2), x.shape[1]).squeeze(2)


# %%
import json
import re
import requests
from bs4 import BeautifulSoup
# %%
class wikipediaProvinceScraper():
    def __init__(self, link: str) -> None:
        self.rsp = requests.get(link)
        if self.rsp.status_code == 200:
            self.soup = BeautifulSoup(self.rsp.text, "html.parser")
        else:
            raise ValueError("Error !")
    def proces(self):
        self.mainLink = "https://fa.wikipedia.org/"
        self.pagesLink = []
        self.tags = self.soup.findAll("div", {"class": "CategoryTreeItem"})
        for i in self.tags:
            self.pagesLink.append(i.findNext("a")["href"])

        self.provinces = []
        for i in self.pagesLink:
            self.tempRsp = requests.get(self.mainLink + i)
            if self.tempRsp.status_code == 200:
                self.tempSoup = BeautifulSoup(self.tempRsp.text, "html.parser")
                self.tempTags = self.tempSoup.find("div", {"class": "mw-category-generated"}).findAll("div", {"class": "mw-category-group"})
                for category in self.tempTags:
                    self.liTags = category.findAll("li")
                    for li in self.liTags:
                        self.provinces.append(li.find("a")["title"])
            else:
                raise ValueError("Error !")

        return self.provinces
# %%
m = wikipediaProvinceScraper("https://fa.wikipedia.org/wiki/%D8%B1%D8%AF%D9%87:%D9%85%D8%AD%D9%84%D9%87%E2%80%8C%D9%87%D8%A7%DB%8C_%D8%AA%D9%87%D8%B1%D8%A7%D9%86_%D8%A8%D8%B1_%D9%BE%D8%A7%DB%8C%D9%87_%D9%85%D9%86%D8%B7%D9%82%D9%87")
mahaleh = (m.proces())
with open("./mahaleh.txt", "w", encoding="utf-8") as f:
    for i in mahaleh:
        f.writelines(i)
        f.writelines("\n")
    f.close()
# %%
with open("./data.json", "r", encoding="utf-8") as f:
    data = json.load(f)
# %%
class province():
    def __init__(self) -> None:
        with open("./stopWords.txt", "r", encoding="utf-8") as f:
            self.stopWords = f.read().splitlines()
            f.close()
        with open("./mahaleh.txt", "r", encoding="utf-8") as f:
            self.mahaleh = f.read().splitlines()
            f.close()
        with open("./reqCount.txt", "r+", encoding="utf-8") as f:
            self.reqCount = int(f.read())
            f.close()
        if self.reqCount >= 5000:
            raise ValueError(f"number of requests to api.neshan.org is more than 5000 !")

    def __del__(self):
        with open("./reqCount.txt", "w", encoding="utf-8") as f:
            f.write(str(self.reqCount))
            f.close()

    def getLngLat(self, url: str):
        rsp = requests.get(url)
        if rsp.status_code == 200:
            try:
                soup = BeautifulSoup(rsp.text, "html.parser")
                lat, long = soup.find("div", {"class": "map-container"}).find("a")["href"].split("/")[-1].split(",")
                return lat.strip(), long.strip()
            except:
                raise ValueError(f"Error in '{url}' for give lat, long !")
        else:
            raise ValueError(f"request to '{url}' || status code: {rsp.status_code}")

    def proces(self, address: str, name: str, link: str):
        apiCode = ""
        if len(address.strip()) == 0 or len(name.strip()) == 0 or len(link.strip()) == 0:
            raise ValueError(f"address: '{address}' is empty OR name: '{name}' is empty OR link: '{link}' is empty")
        else:
            if re.findall(r"\(\w+\)", name):
                return re.findall(r"\(\w+\)", name)[0].strip("(").strip(")")

            for word in self.stopWords:
                rePattern = re.compile(fr"{word.strip()}")
                address = rePattern.sub(" ", address)

            if "،" in address:
                address = address.split("،")
                if len(address[0].strip().split()) > 2:
                    if " ".join(address[0].split()[0:2]) not in self.mahaleh:
                        try:
                            lat, lng = self.getLngLat(link)
                            rsp = requests.get(f"https://api.neshan.org/v5/reverse?lat={lat}&lng={lng}", headers={"Api-Key": f"{apiCode}"})
                            self.reqCount += 1
                            if rsp.status_code == 200:
                                try:
                                    return rsp.json()["neighbourhood"]
                                except:
                                    return -1
                            else:
                                return f"status code: {rsp.status_code} from .neshan.org"

                        except Exception as e:
                            return e

                    else:
                        return " ".join(address[0].split()[0:2])
                else:
                    if len(address[0].strip().split()) == 2:
                        if address[0].strip().split()[0] in self.mahaleh:
                            return address[0].strip().split()[0]
                        elif address[0].strip().split()[1] in self.mahaleh:
                            return address[0].strip().split()[1]
                        else:
                            try:
                                lat, lng = self.getLngLat(link)
                                rsp = requests.get(f"https://api.neshan.org/v5/reverse?lat={lat}&lng={lng}", headers={"Api-Key": f"{apiCode}"})
                                self.reqCount += 1
                                if rsp.status_code == 200:
                                    try:
                                        return rsp.json()["neighbourhood"]
                                    except:
                                        return -1
                                else:
                                    return f"status code: {rsp.status_code} from api.neshan.org"

                            except Exception as e:
                                return e
                    else:
                        if address[0].strip() in self.mahaleh:
                            return address[0].strip()
                        else:
                            try:
                                lat, lng = self.getLngLat(link)
                                rsp = requests.get(f"https://api.neshan.org/v5/reverse?lat={lat}&lng={lng}", headers={"Api-Key": f"{apiCode}"})
                                self.reqCount += 1
                                if rsp.status_code == 200:
                                    try:
                                        return rsp.json()["neighbourhood"]
                                    except:
                                        return -1
                                else:
                                    return f"status code: {rsp.status_code} from api.neshan.org"

                            except Exception as e:
                                return e
            else:
                return " ".join(address.split()[0:2])
# %%
p = province()
prov = []
for city in data:
    for coffee in data[city]:
        ans =  p.proces(data[city][coffee]["address"], coffee, data[city][coffee]["link"])
        prov.append(ans)
        data[city][coffee]["province"] = ans

del p # important!
# %%
def exportTojson(data):
    try:
        with open("./data.json", "w", encoding="utf-8") as f:
            json.dump(data, f, ensure_ascii=False, indent=4)
    except:
        raise ValueError("Can Not Save !")
# %%
exportTojson(data)


from typing import TYPE_CHECKING, Optional, Tuple

from sydent.users.accounts import Account

if TYPE_CHECKING:
    from sydent.sydent import Sydent

class AccountStore:
    def __init__(self, sydent: "Sydent") -> None:
        self.sydent = sydent

    def getAccountByToken(self, token: str) -> Optional[Account]:
        """
        Select the account matching the given token, if any.

        :param token: The token to identify the account, if any.

        :return: The account matching the token, or None if no account matched.
        """
        cur = self.sydent.db.cursor()
        res = cur.execute(
            "select a.user_id, a.created_ts, a.consent_version from accounts a, tokens t "
            "where t.user_id = a.user_id and t.token = ?",
            (token,),
        )

        row: Optional[Tuple[str, int, Optional[str]]] = res.fetchone()
        if row is None:
            return None

        return Account(*row)

    def storeAccount(
        self, user_id: str, creation_ts: int, consent_version: Optional[str]
    ) -> None:
        """
        Stores an account for the given user ID.

        :param user_id: The Matrix user ID to create an account for.
        :param creation_ts: The timestamp in milliseconds.
        :param consent_version: The version of the terms of services that the user last
            accepted.
        """
        cur = self.sydent.db.cursor()
        cur.execute(
            "insert or ignore into accounts (user_id, created_ts, consent_version) "
            "values (?, ?, ?)",
            (user_id, creation_ts, consent_version),
        )
        self.sydent.db.commit()

    def setConsentVersion(self, user_id: str, consent_version: Optional[str]) -> None:
        """
        Saves that the given user has agreed to all of the terms in the document of the
        given version.

        :param user_id: The Matrix ID of the user that has agreed to the terms.
        :param consent_version: The version of the document the user has agreed to.
        """
        cur = self.sydent.db.cursor()
        cur.execute(
            "update accounts set consent_version = ? where user_id = ?",
            (consent_version, user_id),
        )
        self.sydent.db.commit()

    def addToken(self, user_id: str, token: str) -> None:
        """
        Stores the authentication token for a given user.

        :param user_id: The Matrix user ID to save the given token for.
        :param token: The token to store for that user ID.
        """
        cur = self.sydent.db.cursor()
        cur.execute(
            "insert into tokens (user_id, token) values (?, ?)",
            (user_id, token),
        )
        self.sydent.db.commit()

    def delToken(self, token: str) -> int:
        """
        Deletes an authentication token from the database.

        :param token: The token to delete from the database.
        """
        cur = self.sydent.db.cursor()
        cur.execute(
            "delete from tokens where token = ?",
            (token,),
        )
        deleted = cur.rowcount
        self.sydent.db.commit()
        return deleted


import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import skimage.io
import functools
from loguru import logger

logger.info('Import OK')

input_folder = f'SEM-internal-cell-texture/python_results/SEM-internal-cell-texture_Analyzed/feature_collection/'
output_folder = f'SEM-internal-cell-texture/python_results/SEM-internal-cell-texture_Analyzed/summary_calculations/'
scale_folder = f'SEM-internal-cell-texture/python_results/SEM-internal-cell-texture_Analyzed/napari_scale/'

if not os.path.exists(output_folder):
    os.mkdir(output_folder)

# --------------Initialise file lists--------------
# read in calculated pixel data
features = pd.read_csv(f'{input_folder}feature_summary.csv')
features.drop([col for col in features.columns.tolist() if 'Unnamed: ' in col], axis=1, inplace=True)
image_scaled = pd.read_csv(f'{scale_folder}scale_assigned.csv')

# --------------Add pixes per µm factor--------------
# make pixel per µm dictionary
image_scaled_dict = dict(
    zip(image_scaled['image_name'], image_scaled['scaled_pixels_um']))

# map dictionary to features df
features['scaled_pixels_um'] = features['image_name'].map(image_scaled_dict)

features['pixels_nm'] = features['scaled_pixels_um']/1000

# scale: area and axes lengths
features['area_nm'] = (features['area'] / features['pixels_nm'])
features['major_axis_length_nm'] = (
    features['major_axis_length'] / features['pixels_nm'])
features['minor_axis_length_nm'] = (
    features['minor_axis_length'] / features['pixels_nm'])

# --------------Grab major and minor_axis_length for knobs--------------
minor_axis = features[features['ROI_type'] == 'knob'].groupby(
    ['image_name']).mean()['minor_axis_length_nm']
major_axis = features[features['ROI_type'] == 'knob'].groupby(
    ['image_name']).mean()['major_axis_length_nm']

# ----------Calculate average size of knobs per cell----------
av_size = features[features['ROI_type'] == 'knob'].groupby(
    ['image_name']).mean()['area_nm'].reset_index()

# -----------Calculate proportion of area in 'knobs'-----------
knob_area = features[features['ROI_type'] == 'knob'].groupby(['image_name']).sum()['area_nm']

# --------------Calculate number of 'knobs' per cell--------------
# NOTE: due to focus issues, cell shape changes etc, this measure is likely not valid. Generating it just in case!
knob_count = features[features['ROI_type'] == 'knob'].groupby(['image_name']).count()['area_nm']
knobs_per_area = (knob_count / (10000)).reset_index().rename(columns={'area_nm': 'knobs_per_100_nm2'})

# ------------------Summarise, save to csv------------------
summary = functools.reduce(lambda left, right: pd.merge(left, right, on=['image_name'], how='outer'), [av_size, knob_area.reset_index(), knob_count.reset_index(), knobs_per_area, minor_axis, major_axis])
summary.columns = ['image_name', 'mean_knob_area', 'total_knob_area', 'knob_count', 'knobs_per_100_nm2', 'mean_minor_axis', 'mean_major_axis']

# make 'cell_line' column
summary['cell_line'] = summary['image_name'].str.split('-').str[1]

summary.to_csv(f'{output_folder}calculated_properties_summary.csv')

# -------------------visualise calculated parameters-------------------
palette_ko = {
    'CS2': 'midnightblue',
    'OC03': 'palevioletred',
}

for parameter in ['mean_knob_area', 'knob_count', 'knobs_per_100_nm2', 'mean_minor_axis', 'mean_major_axis']:
    fig, ax = plt.subplots()
    sns.swarmplot(x='cell_line', y=parameter,
                  data=summary, palette=palette_ko)
    plt.xlabel('cell_line')
    plt.ylabel(parameter)
    plt.show()

sns.histplot(x=summary['mean_major_axis'],
             kde=True, hue=summary['cell_line'], palette=palette_ko)
plt.show()