# -*- coding: utf-8 -*-

import os
import pickle
import sqlite3
import sys
from time import sleep

from PyQt5.QtCore import QUrl
from PyQt5.QtGui import QDesktopServices, QIcon
from PyQt5.QtWidgets import QApplication
from qfluentwidgets import (ScrollArea, FluentIcon, isDarkTheme, Theme, ToolTipFilter)

from app.common.log import logs
from app.common.my_icons import MyIcon
from app.core.ed2k_decode import convert_bytes
from app.view.custom_view.UI_ed2k_interface import Ui_Form
from my_fluent_widget import show_infoBar
from ..common.config import cfg, FEEDBACK_URL, PyQt_Fluent_Widgets_URL, GITHUB_URL
from ..common.signal_bus import signalBus
from ..common.style_sheet import myStyleSheet

class ed2kInterface(ScrollArea):
    def __init__(self, parent=None):
        super().__init__(parent=parent)

        self.ui = Ui_Form()
        self.ui.setupUi(self)

        self.adjustTableColumns()

        '''connect signal to slot'''
        self.ui.search_lineEdit.searchSignal.connect(self.search_ed2k)
        self.ui.search_lineEdit.clearSignal.connect(lambda: [self.ui.tableWidget.showRow(
            i) for i in range(self.ui.tableWidget.rowCount())])

        self.ui.treeView.clicked.connect(self.read_txt_to_table)
        self.ui.treeView.editSignal.connect(
            lambda: show_infoBar(self, self.tr('提示'), self.tr('打开成功！')))
        self.ui.treeView.refleshSignal.connect(
            lambda: show_infoBar(self, self.tr('提示'), self.tr('刷新文件列表成功')))

        self.ui.tableWidget.copySignal.connect(self.copy_ed2k)
        self.ui.tableWidget.copySignal.connect(
            lambda: setattr(self, 'ed2k_info_mode', 'dragTable'))
        self.ui.tableWidget.addCompleteSignal.connect(self.adjustTableColumns)
        self.ui.tableWidget.addCompleteSignal.connect(self.set_item_toopTip)

        cfg.themeChanged.connect(self.__setQss)

        '''init widget'''  # //DONE  修正tooltip
        self.ui.change_theme_toolbutton.installEventFilter(
            ToolTipFilter(self.ui.change_theme_toolbutton))
        self.ui.change_theme_toolbutton.setIcon(FluentIcon.CONSTRACT)
        self.ui.change_theme_toolbutton.setToolTip(self.tr('切换主题'))
        self.ui.change_theme_toolbutton.clicked.connect(self.toggleTheme)

        self.ui.feedback_toolbutton.installEventFilter(
            ToolTipFilter(self.ui.feedback_toolbutton))
        self.ui.feedback_toolbutton.setIcon(FluentIcon.FEEDBACK)
        self.ui.feedback_toolbutton.setToolTip(self.tr('提供反馈'))
        self.ui.feedback_toolbutton.clicked.connect(
            lambda: QDesktopServices.openUrl(QUrl(FEEDBACK_URL)))

        self.ui.document_pushbutton.setIcon(QIcon(':/gallery/images/logo.png'))
        self.ui.document_pushbutton.clicked.connect(
            lambda: QDesktopServices.openUrl(QUrl(PyQt_Fluent_Widgets_URL)))
        self.ui.document_pushbutton.installEventFilter(ToolTipFilter(
            self.ui.document_pushbutton))
        self.ui.document_pushbutton.setToolTip(self.tr(f'本工具基于此项目'))

        self.ui.github_pushbutton.setIcon(FluentIcon.GITHUB)
        self.ui.github_pushbutton.clicked.connect(
            lambda: QDesktopServices.openUrl(QUrl(GITHUB_URL)))

        self.ui.database_settings_pushButton.installEventFilter(ToolTipFilter(self.ui.database_settings_pushButton))
        self.ui.database_settings_pushButton.setIcon(MyIcon.DATABASE_SETTING)
        self.ui.database_settings_pushButton.setToolTip(self.tr(f'数据库设置'))
        self.ui.database_settings_pushButton.clicked.connect(
            lambda: signalBus.switchToSampleCard.emit('ed2kInterface', 0))

        self.__getConfig()
        myStyleSheet.ED2K_INTERFACE.apply(self)
        self.__setQss()
        self.last_time()
        self.__setSpacing()

    def last_time(self):
        __path =  os.path.join(os.path.dirname(sys.argv[0]), 'data/.ephemeral')
        if os.path.exists(__path):
            with open(__path, 'rb') as f:
                last__time = pickle.load(f)
                self.ui.fixed_label.setText(
                    self.tr(f'欢迎回来，上次使用时间：') + '\n' + last__time)

    def __getConfig(self):
        self.ui.img_checkBox.toggled.connect(lambda:
                                             cfg.set(cfg.checkImageMode, self.ui.img_checkBox.isChecked()))
        self.ui.deduplicate_checkBox.toggled.connect(lambda:
                                                     cfg.set(cfg.checkDeduplicate,
                                                             self.ui.deduplicate_checkBox.isChecked()))
        self.ui.database_checkBox.toggled.connect(self.read_sqlite
                                                  )

        self.ui.img_checkBox.setChecked(cfg.get(cfg.checkImageMode))
        self.ui.deduplicate_checkBox.setChecked(cfg.get(cfg.checkDeduplicate))
        self.ui.database_checkBox.setChecked(cfg.get(cfg.checkDatabase))

    def search_ed2k(self):
        target = self.ui.search_lineEdit.text()
        # 遍历表格中的所有单元格
        for i in range(self.ui.tableWidget.rowCount()):
            item = self.ui.tableWidget.item(i, 0)
            if target.lower() in item.text().lower():
                # 如果找到匹配的单元格，将其设置为当前单元格
                self.ui.tableWidget.showRow(i)
            else:
                self.ui.tableWidget.hideRow(i)

        n = self.ui.tableWidget.rowCount()
        for i in range(n):
            if self.ui.tableWidget.isRowHidden(i):
                n -= 1
        logs.info(f'搜索 {target} 完成, 包含{n}条结果')
        show_infoBar(self, self.tr(f'搜索完成'), self.tr(
            '包含') + str(n) + self.tr('条结果'), type='success')

    def read_sqlite(self):
        state = self.ui.database_checkBox.isChecked()
        cfg.set(cfg.checkDatabase, state)

        if not state:
            return

        conn = sqlite3.connect( os.path.join(os.path.dirname(sys.argv[0]), 'data/database.db'))

        c = conn.cursor()

        c.execute(
            "SELECT name FROM sqlite_master WHERE type='table' AND name='local_data'")
        if c.fetchone():
            c.execute('SELECT * FROM local_data')
            data = c.fetchall()
            data_list = [list(row) for row in data]
            self.ui.tableWidget.add_list_data(data_list)

            logs.info(self.tr('连接数据库成功！'))
            show_infoBar(self, self.tr('提示'), self.tr(
                '连接数据库成功！'), type='success')
            logs.info(self.tr('读取数据成功！'))

        else:
            sleep(1)
            logs.info(self.tr('数据库为空，请先扫描数据'))
            show_infoBar(self, self.tr('错误'), self.tr(
                '数据库为空，请先扫描数据'), type='error')

        conn.close()

    def read_txt_to_table(self):

        if self.ui.database_checkBox.isChecked():
            self.ui.database_checkBox.setChecked(False)

        data = self.ui.treeView.data

        if data:
            path = data[0][6]
            self.ui.tableWidget.setRowCount(0)
            self.ui.tableWidget.add_list_data(data)
            logs.info(f'文件操作：{path}  完成！')
        else:
            logs.debug(f'读取失败，点击文件夹或发生未知错误')

    def copy_ed2k(self):
        '''copy clicked'''

        clipboard_ed2k_list = []

        # Get the row indices of selected rows
        selected_indexes = self.ui.tableWidget.selectedIndexes()
        selected_rows = list(set(index.row() for index in selected_indexes))
        selected_rows.sort()

        for i in selected_rows:
            ed2k = self.ui.tableWidget.item(i, 5).text()
            clipboard_ed2k_list.append(ed2k)

        text = '\n'.join(clipboard_ed2k_list)
        QApplication.clipboard().setText(text)
        show_infoBar(self, self.tr('提示'), self.tr('复制成功！'))
        logs.info(f'复制内容成功：\n{text}')

    def adjustTableColumns(self):
        hide = [2, 4, 5, 7]
        for i in hide:
            self.ui.tableWidget.hideColumn(i)

        if self.ui.tableWidget.columnWidth(0) < 200:
            self.ui.tableWidget.setColumnWidth(0, 200)
        width = self.ui.tableWidget.columnWidth(0)
        # self.ui.tableWidget.setColumnWidth(0, width * 0.3)
        self.ui.tableWidget.setColumnWidth(1, width * 0.5)
        self.ui.tableWidget.setColumnWidth(6, width * 0.5)

    def set_item_toopTip(self):
        for row in range(self.ui.tableWidget.rowCount()):
            for column in [0, 6]:
                item = self.ui.tableWidget.item(row, column)

                if item:
                    # //DONE  优化样式
                    item.setToolTip(item.text())

        row_number = self.ui.tableWidget.rowCount()
        sum_byte = self.ui.tableWidget.sum_col_numbers(2)
        sum = convert_bytes(sum_byte)

        logs.info(f'计算文件大小：{sum}    字节数：{sum_byte}  ')
        self.ui.label.setText(self.tr('文件数量：') + str(row_number))
        self.ui.label_2.setText(self.tr('文件大小：') + sum)
        self.__setSpacing()

    def toggleTheme(self):
        theme = Theme.LIGHT if isDarkTheme() else Theme.DARK
        cfg.set(cfg.themeMode, theme)

        theme = 'dark' if isDarkTheme() else 'light'
        logs.info(f'切换{theme} 主题成功')
        show_infoBar(self, self.tr('操作成功'), self.tr(
            f'已切换 ') + theme + self.tr(' 主题'), type='success')

        self.__setQss()

    def __setQss(self):
        """ set style sheet """

        if isDarkTheme():
            style = self.ui.tableWidget.styleSheet()
            style += " QToolTip { background-color: #333333; color: white; border: none;font: 13px 'Microsoft YaHei'; font-weight: bold;  }"
        else:
            style = self.ui.tableWidget.styleSheet()
            style += " QToolTip { font: 13px 'Microsoft YaHei'; font-weight: bold; }"
        self.ui.tableWidget.setStyleSheet(style)

    def __setSpacing(self):
        width = self.ui.frame_2.width()

        if width < 215:
            self.ui.fixed_label.setFixedWidth(215)
            return
        self.ui.fixed_label.setFixedWidth(width + 2)

    def resizeEvent(self, e):
        super().resizeEvent(e)
        self.__setSpacing()


# check_eigvectors.py

import numpy as np
from math import pi
import matplotlib.pyplot as plt
from create_hamiltonian import HamiltonianTilt

if __name__ == "__main__":

    case = "check eigvecs"

    if case == "check eigvecs":

        N = 1
        M = 3
        t = 1
        a = 1
        b = 4 * pi / 3 / a
        delta_nnn = np.zeros((N, M, 2, 3))

        t_bar = 0
        t_amp = 0.1
        p = 1
        try:
            t_freq = 2 * pi * p / a / (M - 1)
        except:
            t_freq = 1
        delta_nn = HamiltonianTilt.generate_nn_vary_ttilda(N, M, t_bar, t_amp, t_freq, a)

        ham = HamiltonianTilt(N, M, t, delta_nn, delta_nnn, a)

        k1 = ham.k1
        k2 = np.array([-np.sqrt(3) / 2 * b, -b / 2])

        k_shift = 2 / 6 * (k1 + k2)

        k_max = 2/20
        k_num = 2000
        k_list = [k_shift + k_max * (i-k_num/2) / k_num * (k1 + k2) for i in range(k_num)]
        spectrum = []

        for i in range(k_num):
            ham.construct_ham_inK(k_list[i])
            ev = np.linalg.eigvals(ham.H)
            ev = np.sort(ev)
            spectrum.append(ev)

        spectrum = np.array(spectrum)

        fig, ax = plt.subplots()
        ax.plot(spectrum)

        plt.show()


from bs4 import BeautifulSoup
import requests
import pandas as pd
import time
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from urllib.request import urlretrieve
import os

emme_url = 'https://5101137.onlineleasing.realpage.com/'

# initialize the driver and open the Chrome browser
driver = webdriver.Chrome('/Users/adouglas/Desktop/chromedriver')

# open the webpage in the current driver
driver.get(emme_url)
time.sleep(10)

WebDriverWait(driver, 60).until(EC.frame_to_be_available_and_switch_to_it(driver.find_element_by_xpath('//iframe[@id="rp-leasing-widget"]')))

# first get the floorlans
current_floorplans = os.listdir('/Users/adouglas/Google Drive/real_estate/chicago_listing_scraping/floorplans_emme')

rows = WebDriverWait(driver, 60).until(EC.presence_of_all_elements_located((By.CLASS_NAME, 'floorplan-tile')))

for row in rows:
	row_soup = BeautifulSoup(row.get_attribute('innerHTML'), "html.parser")
	img_div = row_soup.find('div', class_='loaded-image')
	plan_url = img_div.attrs['style'].split('(')[1].split(')')[0].replace('"','').replace('320x320','1024x1024')
	file_name = row_soup.find('span', class_='specs').text.split('|')[0].replace(' ','_') + \
				row_soup.find('span', class_='specs').text.split('|')[1].strip().replace(' ','_') + \
				row_soup.find('span', class_='specs').text.split('|')[2].replace(' ','_') + plan_url[-4:]
	if file_name not in current_floorplans:
		urlretrieve(plan_url, '/Users/adouglas/Google Drive/real_estate/chicago_listing_scraping/floorplans_emme/' + file_name)

WebDriverWait(driver, 60).until(EC.presence_of_element_located((By.TAG_NAME, 'button')))
time.sleep(5)

rows = driver.find_elements_by_tag_name('button')

if rows[0].get_attribute('innerHTML') == 'Start':
	rows[0].click()

elem = driver.find_element_by_class_name('help-div')
elem.find_element_by_tag_name('a').click()

rows = driver.find_elements_by_tag_name('button')
row_count = len(rows)

i = 0

for i in range(row_count):
	rows = WebDriverWait(driver, 60).until(EC.presence_of_all_elements_located((By.TAG_NAME, 'button')))

	if 'Available' in rows[i].get_attribute('innerHTML'):
		rows[i].click()

		unit_data = WebDriverWait(driver, 60).until(EC.presence_of_element_located((By.CLASS_NAME, 'data-table')))
		data_soup = BeautifulSoup(unit_data.get_attribute('innerHTML'), "html.parser")
		beds  = data_soup.text.split('Bedrooms')[1].split('Bathrooms')[0]
		baths = data_soup.text.split('Bathrooms')[1].split('Square')[0]
		unit_sf = data_soup.text.split('Feet')[1].split('Occupancy')[0]

		available_rows = driver.find_elements_by_class_name('radio-option')
		for available_unit in available_rows:
			if 'aria-label' in available_unit.get_attribute('innerHTML'):
				row_soup = BeautifulSoup(available_unit.get_attribute('innerHTML'), "html.parser")
				unit_num  = row_soup.find('span', class_='unit-cell').text
				unit_rent = row_soup.find('span', class_='unit-price').text
				print(beds, baths, unit_sf, unit_num, unit_rent)

		back_button = driver.find_element_by_id('footer-back-button')
		back_button.click()
		i += 1
	else:
		i += 1

driver.close()


import matplotlib.pyplot as plt
import pandas as pd
import numpy as np

clear_sky_days = ['2019-09-02', '2019-09-03', '2019-09-24', '2019-09-25',
                  '2019-09-26', '2019-09-27', '2019-09-28', '2019-09-29',
                  '2019-10-04', '2019-10-09', '2019-10-13', '2019-10-15',
                  '2019-10-16', '2019-10-24', '2019-10-25', '2019-10-26',
                  '2019-10-29', '2019-11-04', '2019-11-05', '2019-11-18',
                  '2019-11-27', '2019-11-29', '2019-11-30']

len(clear_sky_days)
overcast_days = ['2019-10-22']

df = pd.read_csv('Chu_Data.csv', parse_dates=True , index_col='Forecating Starting Time UTC')

df['day'] = df.index.day
df['month'] = df.index.month
df_clear = pd.DataFrame()

for days in clear_sky_days:
    day = int(days.split('-')[2])
    month = int(days.split('-')[1])
    df_day = df[(df['day'] == day) & (df['month'] == month)]
    df_clear = df_clear.append(df_day)

df_clear['RM_diff'].hist(bins=1000)

x1 = np.sort(df_clear['RM_diff'])
N = len(df_clear['RM_diff'])
y = np.arange(N)/float(N)

q_95 = df_clear['RM_diff'].quantile(0.95)
q_75 = df_clear['RM_diff'].quantile(0.75)
q_99 = df_clear['RM_diff'].quantile(0.99)

plt.rcParams["font.weight"] = "bold"
plt.rcParams["axes.labelweight"] = "bold"

f, ax = plt.subplots(figsize = (12, 8))
plt.plot(x1, y, marker='o', color="k", markersize=8)
plt.axhline(y=0.95 , color='red', ls = "--")
plt.axvline(q_95 ,color='red',  ls = "--")
plt.text(q_95, 0.99, "ThrB =(" + str(round(q_95,4)) + ", " + str(0.95) + ")", color = 'tab:green', fontsize=14)
plt.axhline(y=0.75 , color='red', ls = "--")
plt.axvline(q_75 ,color='red',  ls = "--")
plt.text(q_75, 0.77, "ThrA =(" + str(round(q_75,4)) + ", " + str(0.75) + ")", color = 'tab:green', fontsize=14)
plt.ylabel("F(x)", fontsize= 24, fontweight= 'bold')
plt.xlabel('RM_diff', fontsize= 24,style='italic', fontweight= 'bold')
plt.tick_params(axis='y', labelsize=20)
plt.tick_params(axis='x', labelsize=20)
plt.title('cleark-sky days', fontsize=25, style='italic', fontweight='bold', y=1.01)
plt.savefig("CDF_clr_days.png", format = "png", dpi = 150, bbox_inches = "tight")
plt.close()

plt.rcParams["font.weight"] = "bold"
plt.rcParams["axes.labelweight"] = "bold"

for minute in df_clear['minute'].unique():

    f, ax = plt.subplots(figsize = (12, 8))
    plt.hist(df_clear['RM_diff'], bins=300)
    plt.ylabel("Counts", fontsize= 24, fontweight= 'bold')
    plt.xlabel('RM_diff', fontsize= 24,style='italic', fontweight= 'bold')
    plt.tick_params(axis='y', labelsize=20)
    plt.tick_params(axis='x', labelsize=20)
    plt.title('Clear-sky days', fontsize=25, style='italic', fontweight='bold', y=1.01)
    plt.savefig("Hist_clr_days.png", format = "png", dpi = 150, bbox_inches = "tight")
    plt.close()

plt.rcParams["font.weight"] = "bold"
plt.rcParams["axes.labelweight"] = "bold"
thr = []
for minute in df_clear['minute'].unique():

    df_minute = df_clear['RM_diff'][df_clear['minute'] == minute]
    q_95 = df_minute.quantile(0.95)
    q_75 = df_minute.quantile(0.75)
    q_99 = df_minute.quantile(0.99)

    thr.append({'q75':q_75, 'q95':q_95,'q99':q_99})
thr = pd.DataFrame(thr, index = df_clear['minute'].unique())
thr.plot()

thr.to_csv('Thr_Chu.csv')

f, ax = plt.subplots(figsize = (12, 8))
plt.errorbar(thr.index, thr['q75'], thr['q75']*0.1, lw=2, label = 'q75')
plt.errorbar(thr.index, thr['q95'], thr['q95']*0.1, lw=2, label = 'q95')
plt.errorbar(thr.index, thr['q99'], thr['q99']*0.1, lw=2, label = 'q99')
plt.tick_params(axis='y', labelsize=20)
plt.tick_params(axis='x', labelsize=20)
plt.xticks(thr.index)
plt.legend(loc='upper left')
plt.savefig("Thr.png", format = "png", dpi = 150, bbox_inches = "tight")
plt.close()


from django.test import TestCase
from rest_framework.test import APIRequestFactory
from rest_framework import status
from ..models import User
from ..views import user_list

class TestUserViews(TestCase):

    def setUp(self):
        self.factory = APIRequestFactory()
        self.test_user1 = User.objects.create(username='test_user1', age=25)
        self.test_user2 = User.objects.create(username='test_user2', age=30)

    def test_create_user(self):
        data = {'username': 'new_test_user', 'age': 28}
        request = self.factory.post('/api/users/', data, format='json')
        response = user_list(request)
        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
        self.assertEqual(User.objects.count(), 3)

    def test_get_user(self):
        request = self.factory.get(f'/api/users/{self.test_user1.id}/')
        response = user_list(request)
        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.context['users'][0].username, self.user1.username)

    def test_delete_user(self):
        request = self.factory.delete(f'/api/users/{self.test_user2.id}/')
        response = user_list(request)
        self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
        self.assertEqual(User.objects.count(), 1)


import torch
import torch.nn as nn
import numpy as np
import sys

from model import model
from tqdm import tqdm
from utils.helpers import draw_seg_maps
from utils.helpers import save_model_dict
from utils.metrics import eval_metric
from utils.helpers import TensorboardWriter

class Trainer:
    def __init__(self, model, train_data_loader, train_dataset,
                 valid_data_loader, valid_dataset, classes_to_train,
                 epochs, device, lr, resume_training=None, model_path=None):
        super(Trainer, self).__init__()

        self.train_data_loader = train_data_loader
        self.train_dataset = train_dataset
        self.valid_data_loader = valid_data_loader
        self.valid_dataset = valid_dataset
        self.model = model
        self.num_classes = len(classes_to_train)
        self.epochs = epochs
        self.device = device
        self.lr = lr

        self.optimizer = torch.optim.Adam(model.parameters(), lr=self.lr)
        print('OPTIMIZER INITIALIZED')
        self.criterion = nn.CrossEntropyLoss()
        print('LOSS FUNCTION INITIALIZED')

        # initialize Tensorboard `SummaryWriter()`
        self.writer = TensorboardWriter()

        print(f"NUM CLASSES: {self.num_classes}")

        if resume_training == 'yes':
            print('RESUMING TRAINING')
            # load the model checkpoint
            checkpoint = torch.load(model_path)
            self.trained_epochs = checkpoint['epoch']
            self.train_iters = checkpoint['train_iters']
            self.valid_iters = checkpoint['valid_iters']
            print(f"PREVIOUSLY TRAINED EPOCHS: {self.trained_epochs}")
            if self.trained_epochs >= self.epochs:
                print('Current epochs less than previously trained epcochs...')
                print(f"Please provide greater number of epochs than {self.trained_epochs}")
                sys.exit()
            elif self.epochs > self.trained_epochs:
                #  load model weights state_dict
                 self.model.load_state_dict(checkpoint['model_state_dict'])
                 print('TRAINED MODEL WEIGHTS LOADED...')
                 # load trained optimizer state_dict
                 self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
                 print('TRAINED OPTIMIZER LOADED...')

        elif resume_training == 'no':
            self.train_iters = 0
            self.valid_iters = 0
            self.trained_epochs = 0
            print('TRAINING FROM BEGINNING')

    def get_num_epochs(self):
        return self.trained_epochs

    def fit(self):
        print('Training')
        model.train()
        train_running_loss = 0.0
        train_running_inter, train_running_union = 0, 0
        train_running_correct, train_running_label = 0, 0
        # calculate the number of batches
        num_batches = int(len(self.train_dataset)/self.train_data_loader.batch_size)
        prog_bar = tqdm(self.train_data_loader,
                        total=num_batches)
        counter = 0 # to keep track of batch counter
        for i, data in enumerate(prog_bar):
            counter += 1
            data, target = data[0].to(self.device), data[1].to(self.device)
            self.optimizer.zero_grad()
            outputs = self.model(data)
            outputs = outputs['out']

            ##### BATCH-WISE LOSS #####
            loss = self.criterion(outputs, target)
            train_running_loss += loss.item()
            ###########################

            ##### BATCH-WISE METRICS ####
            correct, labeled, inter, union = eval_metric(outputs,
                                                             target,
                                                             self.num_classes)
            # for IoU
            train_running_inter += inter
            train_running_union += union
            # for pixel accuracy
            train_running_correct += correct
            train_running_label += labeled
            #############################

            ##### BACKPROPAGATION AND PARAMETER UPDATION #####
            loss.backward()
            self.optimizer.step()
            ##################################################

            ##### TENSORBOARD LOGGING #####
            train_running_IoU = 1.0 * inter / (np.spacing(1) + union)
            train_running_mIoU = train_running_IoU.mean()
            train_running_pixacc = 1.0 * correct / (np.spacing(1) + labeled)
            self.writer.tensorboard_writer(
                loss, train_running_mIoU, train_running_pixacc, self.train_iters,
                phase='train'
            )
            ###############################

            prog_bar.set_description(desc=f"Loss: {loss:.4f} | mIoU: {train_running_mIoU:.4f} | PixAcc: {train_running_pixacc:.4f}")

            self.train_iters += 1

        ##### PER EPOCH LOSS #####
        train_loss = train_running_loss / counter
        ##########################

        ##### PER EPOCH METRICS ######
        # IoU and mIoU
        IoU = 1.0 * train_running_inter / (np.spacing(1) + train_running_union)
        mIoU = IoU.mean()
        # pixel accuracy
        pixel_acc = 1.0 * train_running_correct / (np.spacing(1) + train_running_label)
        ##############################
        return train_loss, mIoU, pixel_acc

    def validate(self, epoch):
        print('Validating')
        model.eval()
        valid_running_loss = 0.0
        valid_running_inter, valid_running_union = 0, 0
        valid_running_correct, valid_running_label = 0, 0
        # calculate the number of batches
        num_batches = int(len(self.valid_dataset)/self.valid_data_loader.batch_size)
        with torch.no_grad():
            prog_bar = tqdm(self.valid_data_loader,
                        total=num_batches)
            counter = 0 # to keep track of batch counter
            for i, data in enumerate(prog_bar):
                counter += 1
                data, target = data[0].to(self.device), data[1].to(self.device)
                outputs = self.model(data)
                outputs = outputs['out']

                # save the validation segmentation maps every...
                # ... last batch of each epoch
                if i == num_batches - 1:
                    draw_seg_maps(data, outputs, epoch, i)

                ##### BATCH-WISE LOSS #####
                loss = self.criterion(outputs, target)
                valid_running_loss += loss.item()
                ###########################

                ##### BATCH-WISE METRICS ####
                correct, labeled, inter, union = eval_metric(outputs,
                                                                target,
                                                                self.num_classes)
                valid_running_inter += inter
                valid_running_union += union
                # for pixel accuracy
                valid_running_correct += correct
                valid_running_label += labeled
                #############################

                ##### TENSORBOARD LOGGING #####
                valid_running_IoU = 1.0 * inter / (np.spacing(1) + union)
                valid_running_mIoU = valid_running_IoU.mean()
                valid_running_pixacc = 1.0 * correct / (np.spacing(1) + labeled)
                self.writer.tensorboard_writer(
                    loss, valid_running_mIoU, valid_running_pixacc, self.valid_iters,
                    phase='valid'
                )
                ###############################

                prog_bar.set_description(desc=f"Loss: {loss:.4f} | mIoU: {valid_running_mIoU:.4f} | PixAcc: {valid_running_pixacc:.4f}")

                self.valid_iters += 1

        ##### PER EPOCH LOSS #####
        valid_loss = valid_running_loss / counter
        ##########################

        ##### PER EPOCH METRICS ######
        # IoU and mIoU
        IoU = 1.0 * valid_running_inter / (np.spacing(1) + valid_running_union)
        mIoU = IoU.mean()
        # pixel accuracy
        pixel_acc = 1.0 * valid_running_correct / (np.spacing(1) + valid_running_label)
        ##############################
        return valid_loss, mIoU, pixel_acc

    def save_model(self, epochs):
        save_model_dict(self.model, epochs,
                        self.optimizer, self.criterion,
                        self.valid_iters, self.train_iters)


import os
import xarray as xr
import numpy as np
from odc.geo.xr import assign_crs

def allNaN_arg(da, dim, stat, idx=True):
    """
    Calculate da.argmax() or da.argmin() while handling
    all-NaN slices. Fills all-NaN locations with an
    float and then masks the offending cells.

    Parameters
    ----------
    da : xarray.DataArray
    dim : str
        Dimension over which to calculate argmax, argmin e.g. 'time'
    stat : str
        The statistic to calculte, either 'min' for argmin()
        or 'max' for .argmax()
    idx : bool
        If True then use da.idxmax() or da.idxmin(), otherwise
        use ds.argmax() or ds.argmin()

    Returns
    -------
    xarray.DataArray
    """
    # generate a mask where entire axis along dimension is NaN
    mask = da.isnull().all(dim)

    if stat == "max":
        y = da.fillna(float(da.min() - 1))
        if idx==True:
            y = y.idxmax(dim=dim, skipna=True).where(~mask)
        else:
            y = y.argmax(dim=dim, skipna=True).where(~mask)
        return y

    if stat == "min":
        y = da.fillna(float(da.max() + 1))
        if idx==True:
            y = y.idxmin(dim=dim, skipna=True).where(~mask)
        else:
            y = y.argmin(dim=dim, skipna=True).where(~mask)
        return y

def round_coords(ds):
    """
    Due to precision of float64 on coordinates, coordinates
    don't quite match after reprojection, resulting in adding spurious
    pixels after merge. Converting to float32 rounds coords so they match.
    """
    ds['latitude'] = ds.latitude.astype('float32')
    ds['longitude'] = ds.longitude.astype('float32')

    ds['latitude'] = np.array([round(i,4) for i in ds.latitude.values])
    ds['longitude'] = np.array([round(i,4) for i in ds.longitude.values])

    return ds

def collect_prediction_data(time_start,
                            time_end,
                            scale='1km',
                            covariables=[
                                 #'LAI',
                                 #'LAI_anom',
                                 'kNDVI',
                                 'kNDVI_anom',
                                 #'FPAR',
                                 #'FPAR-NDVI',
                                 'LST',
                                 'trees',
                                 'grass',
                                 'bare',
                                 'C4_grass',
                                 #'Tree',
                                 #'NonTree',
                                 #'NonVeg',
                                 'LST_Tair',
                                 'TWI',
                                 'NDWI',
                                 #'NDWI_anom',
                                 'rain',
                                 'rain_cml3',
                                 'rain_cml6',
                                 'rain_cml12',
                                 'rain_anom',
                                 'rain_cml3_anom',
                                 'rain_cml6_anom',
                                 'rain_cml12_anom',
                                 'srad',
                                 'srad_anom',
                                 'vpd',
                                 'tavg',
                                 'tavg_anom',
                                 #'SOC',
                                 #'CO2',
                                 #'C4percent',
                                 #'Elevation',
                                 #'MOY',
                                 'VegH',
                                 #'MI'
                            ],
                            chunks=dict(latitude=1150, longitude=1100, time=1),
                            export=False,
                            verbose=True
                           ):

    dss=[]
    base='/g/data/os22/chad_tmp/NEE_modelling/data/'
    for var in covariables:
        if verbose:
            print(f'   Extracting {var}')

        ds = xr.open_dataset(f'{base}{scale}/{var}_{scale}_monthly_2002_2022.nc',
                             chunks=chunks
                            )
        ds = ds.sel(time=slice(time_start, time_end))

        #makse sure coords match (remove trailing zeros)
        ds['latitude'] = ds.latitude.astype('float32')
        ds['latitude'] = np.array([round(i,4) for i in ds.latitude.values])
        ds['longitude'] = ds.longitude.astype('float32')
        ds['longitude'] = np.array([round(i,4) for i in ds.longitude.values])

        dss.append(ds)

    #merge all datasets together
    if verbose:
        print('   Merge and create valid data mask')
    data = xr.merge(dss, compat='override')

    # #create mask where data is valid (excludes urban, water)
    # mask = ~np.isnan(data['PFT'].isel(time=0))
    # data = data.where(mask)

    #remove landcover
    #data = data.drop('PFT')

    if verbose:
        print('   Exporting netcdf')

    # export data
    data = data.rename({'latitude':'y', 'longitude':'x'}) #this helps with predict_xr
    data = data.astype('float32') #make sure all data is in float32
    data = assign_crs(data, crs='epsg:4326')

    if export:
        data.to_netcdf('/g/data/os22/chad_tmp/NEE_modelling/results/prediction_data/prediction_data_'+time_start+'_'+time_end+'.nc')

    return data


import io

import requests

from userbot import CMD_HELP
from userbot.events import errors_handler, register

UD_QUERY_URL = 'http://api.urbandictionary.com/v0/define'
UD_RANDOM_URL = 'http://api.urbandictionary.com/v0/random'

@register(outgoing=True, pattern="ud")
@errors_handler
async def urban_dict(event):
    udquery = event.pattern_match.group(1)

    if udquery:
        params = {'term': udquery}
        url = UD_QUERY_URL
    else:
        params = None
        url = UD_RANDOM_URL

    with requests.get(url, params=params) as response:
        if response.status_code == 200:
            response = response.json()
        else:
            await event.edit(f"`An error occurred, response code:` **{response.status_code}**")
            return

    if response['list']:
        response_word = response['list'][0]
    else:
        await event.edit(f"`No results for query:` **{udquery}**")
        return

    definition = f"**{response_word['word']}**: `{response_word['definition']}`"

    if response_word['example']:
        definition += f"\n\n**Example**: `{response_word['example']}`"

    definition = definition.replace("[", "").replace("]", "")

    if len(definition) >= 4096:
        file_io = io.BytesIO()
        file_io.write(bytes(definition.encode('utf-8')))
        file_io.name = f"definition of {response_word['word']}.txt"
        await event.client.send_file(event.chat_id, file_io, caption="`Output was too large, sent it as a file.`")
        await event.delete()
        return

    await event.edit(definition)

CMD_HELP.update({
    'ud':
    ".ud <search_query>"
    "\nUsage: Does a search on Urban Dictionary."
})


from mbclient import mb
import shared
from random import choice
from datetime import datetime
import re

#for later maybe, need to redo how private notices are received for this
#who_will="who(?:\s+and\s+(?P<second>who))?(?P<plural>\s+are|\s+were|'re)?(?P<dowhat>.+)"
eightball=".+"
help="help(?:\s+with)?(?:\s+(?P<cat>\w+))?"
daddy="who'?s?\s+is\s+(?:ur|your)\s+daddy"
parent="who'?s?\s+\is\s+(?:ur|your)\s+(?:father|mother|dad|parent)"
save="save\s+(?P<thing>.+)\s+as\s+(?P<name>.+)"
get="(?:get|fetch|gimme|give)(?:\sme)?\s+(?P<name>.+)|(?:what\s+have\s+you\s+got)"
yiff="yiff\s+(?P<who>.+)"
coin = "choose\s+(?P<what>.+)$"
seen = "(?:have\s+you\s+)?seen\s+(?P<who>\S+)"
send_message="tell\s+(?P<who>\S+)\s+(?P<what>.+)"
time="time"

hi="(?:hi+|he+llo+|he+y|yo+)[!\s\.]*"

thank="(?:tha+nks?\s*(?:you)?)[!\s\.]*"

urwelc=["pay me","love u bb","you're welcome","that's it??? for all this hard work??","what would you boobs do without me"]
hello=["bye","shut up","go away","not you again","hi","hello","uh huh","sigh"]

def coin_func(nick,match,target):
	what = match.group('what')
	p=re.compile("\s*(?:and|or|\.|,)+\s*")
	options = p.split(what)

	mb.tell(nick+ ": "+choice(options),target)

def yiff_pick(match):
	string = match.group(1)
	quick = re.search("\|",string)
	if quick:
		options=string.split("|")
	else:
		options = mb.data['yiff'][string]
	res = choice(options)
	return res

def yiff_func(nick,match,target):
	who=match.group("who")
	if who == "me":
		who = nick
	elif who in ["yourself","urself"]:
		who = mb.nickname
	mb.data['yiff']['caller']=[nick];
	mb.data['yiff']['target']=[who];
	too_much=20
	pattern = re.compile("{([|\w_]+)}")
	yiff = choice(mb.data['yiff']['yiff'])
	while re.search("{",yiff) and too_much>0:
		too_much -= 1
		yiff=re.sub(pattern,yiff_pick,yiff)
	mb.tell(yiff,target,action=True)

def get_func(nick,match,target):
	name=match.group('name')
	if name:
		name=name.lower()
	else:
		name=".*"
	if name in mb.data['data']:
		mb.tell(nick+": "+mb.data['data'][name],target,True)
	else:
		results=[]
		for title, thing in mb.data['data'].items():
			if re.search(name,title+thing, flags=re.IGNORECASE):
				results.append(title)
		if results==[]:
			mb.tell(nick+": dunno anything like that",target)
		else:
			mb.tell(nick+": did you mean "+", ".join(results)+"? you shit? you dumbass?",target)

def save_func(nick,match,target):
	name=match.group('name').lower()
	thing=match.group('thing')
	mb.data['data'][name]=thing
	mb.tell(nick+": aight",target)
	mb.save('data')

def help_func(nick,match,target,param=None):
	category=match.group('cat')
	if category in mb.help:
		mb.tell(nick+": "+mb.help[category],target)
	else:
		mb.tell(nick+ ": help with "+", ".join(list(mb.help.keys())),target)
		pattern=re.compile("^(?P<cat>"+"|".join(list(mb.help.keys()))+")$",flags=re.IGNORECASE)
		response={'pattern':pattern,'nick':nick,'target':target,'func':help_func}
		mb.responses['help']=response

def eightball_func(nick,match,target):
	mb.tell(nick+": "+choice(mb.data['stuff']['premonition']),target)

def time_func(nick,match,target):
	mb.tell(nick+": "+str(datetime.now().time()),target)

def parent_func(nick,match,target):
	mb.tell("Glub is and it fills me with existential dread and urge to vomit",target)

def daddy_func(nick,match,target):
	mb.tell("ew what the fuck is wrong with you",target)

def hi_func(nick,match,target):
	mb.tell(nick+": "+choice(hello),target)

def thank_func(nick,match,target):
	mb.tell(nick+": "+choice(urwelc),target)

def seen_func(nick,match,target):
	who = match.group('who')

	if who in mb.data['logs']:
		date = datetime.now()
		when = mb.data['logs'][who]['date']
		now=shared.time_dict(date)

		time=shared.time_diff(now,when)
		mb.tell(nick+": "+who+" was here "+time+", saying '"+mb.data['logs'][who]['message']+"'",target)
	else:

		mb.tell(nick+": never heard of that douche",target)

def send_message_func(nick,match,target):
	who = match.group('who')
	date = datetime.now()
	when=shared.time_dict(date)

	message = match.group('what')
	if not who in mb.data['logs']:
		mb.data['logs'][who]={}

	if not 'messages' in mb.data['logs'][who]:
		mb.data['logs'][who]['messages']=[]

	mb.data['logs'][who]['messages'].append({'text':message, 'nick': nick,'date':when})

	mb.tell(nick+": jeez FINE I'll tell em",target)
	mb.save('logs')

mb.add_command(send_message,send_message_func)
mb.add_command(seen,seen_func)

mb.add_command(hi,hi_func)
mb.add_command(coin,coin_func)
mb.add_command(thank,thank_func)
#mb.add_command(daddy,daddy_func,priority=0)
#mb.add_command(parent,parent_func)
mb.add_command(yiff,yiff_func)
mb.add_command(get,get_func)
mb.add_command(save,save_func, level=1)
mb.add_command(help,help_func)
mb.add_command(time,time_func)
mb.add_command(eightball,eightball_func,priority=999)
mb.help['choice'] = "mb choose <list of things separated by , 'or' or 'and'>"
mb.help['seen'] = "mb seen <who>"
mb.help['tell'] = "mb tell <who> <what>"

mb.help['8ball']="anything that starts with mb and doesn't fit any other command is treated like an 8ball command"
mb.help['memos']="mb yiff <someone>, mb save <something> as <name> - saves some text under <name>, mb get <name> - retrieves it"
print("loaded misc")


import subprocess
from .brew import Brew

import requests

from .abs_package import AbsPackage
from .util import bin_loc, is_installed

class Kubectl(AbsPackage):
    def __init__(self):
        super().__init__()

    def is_installed(self):
        return is_installed("kubectl")

    def get_version(self):
        output = subprocess.check_output(
            [
                "kubectl",
                "version",
                "--client",
                "--short",
            ]
        )

        output = output.decode("utf-8")
        words = output.split()
        return words[2][1:]

    def linux_install(self):
        latest_release = requests.get(
            "".join(
                [
                    "https://storage.googleapis.com/",
                    "kubernetes-release/release/stable.txt",
                ]
            )
        ).text

        address = "".join(
            [
                "https://storage.googleapis.com/",
                "kubernetes-release/release/",
                latest_release,
                "/bin/linux/amd64/kubectl",
            ]
        )
        self.install_binary_from_address(
            address=address,
            binary_name="kubectl",
        )

    def linux_uninstall(self):
        kubectl_loc = bin_loc("kubectl")
        if kubectl_loc is not None:
            subprocess.run(
                [
                    "sudo",
                    "rm",
                    kubectl_loc,
                ]
            )

    def osx_install(self):
        brew = Brew()
        brew.brew_install(
            pkgs="kubectl",
        )

    def osx_uninstall(self):
        brew = Brew()
        brew.brew_uninstall(
            pkgs="kubectl",
        )


from app import app
from app.controller.product_controller import ProductController
from app.controller.cart_controller import CartController
from flask import request, redirect, url_for, request, flash
from flask_login import current_user, login_required
# this is the routes for the cart. for each function you need to be logged in to use
# this will get the cart controller
@app.route("/get_cart", methods=['GET'])
@login_required
def get_cart():
    return CartController.get_all()
# this will add to the cart
@app.route("/add_to_cart", methods=['POST'])
@login_required
def add_to_cart():
    seed = request.args.get('seed_id')
    quantity = request.args.get('quantity', type=int)
    return CartController.add(seed, quantity)
# removes the item from the cart
@app.route("/removed/<int:item_removed>", methods=['POST'])
@login_required
def removed(item_removed):
    user_n = current_user.id
    qty = request.form.get('qty', type=int)
    seed_stock = ProductController.get_product_quantity(item_removed)
    # checks the stock when removed
    if qty > seed_stock:
        flash('QTY error')
        return redirect(url_for('get_cart'))
    else:
        if qty > 0:
            CartController.update(user_n, item_removed, qty)
        else:
            CartController.delete(user_n, item_removed)
        return redirect(url_for('get_cart'))


import numpy as np
import matplotlib.pyplot as plot
import matplotlib.patches as patches
import data
from sklearn.decomposition import PCA
import pandas as pd

def add_rects(ax):
    rect1 = patches.Rectangle((420, 320),
                              60,
                              30,
                              linewidth=1,
                              edgecolor='r',
                              facecolor='none',
                              label='C')
    ax.add_patch(rect1)

    rect2 = patches.Rectangle((380, 250),
                              100,
                              10,
                              linewidth=1,
                              edgecolor='g',
                              facecolor='none',
                              label='A')
    ax.add_patch(rect2)

    rect3 = patches.Rectangle((380, 310),
                              40,
                              10,
                              linewidth=1,
                              edgecolor='b',
                              facecolor='none',
                              label='M')
    ax.add_patch(rect3)

    rect4 = patches.Rectangle((300, 270),
                              20,
                              10,
                              linewidth=1,
                              edgecolor='y',
                              facecolor='none',
                              label='B')
    ax.add_patch(rect4)

    rect5 = patches.Rectangle((320, 270),
                              30,
                              10,
                              linewidth=1,
                              edgecolor='w',
                              facecolor='none',
                              label='T')
    ax.add_patch(rect5)

def get_local_max(data, koeff, X, Y):
    local_max = []
    x, y = data.shape
    rad = 10

    for i in range(x):
        for j in range(y):
            l = i - rad if i - rad > 0 else 0
            r = i + rad if i + rad < x else x
            u = j - rad if j - rad > 0 else 0
            d = j + rad if j + rad < y else y
            slice = data[l:r, u:d]

            if data[i][j] == np.max(slice) and data[i][j] > koeff:
                local_max.append([X[j], Y[i], data[i][j]])

    return np.array(local_max).T

def parse_line(line):
    line = line.replace('\n', '').split('\t')
    data = []
    for number in line:
        if number == '':
            data.append(0)
        elif not number.isdigit() and not "," in number:
            continue
        else:
            data.append(float(number.replace(',', '.')))

    return data

def del_ryley(Z, X, Y):
    for i in range(len(Y)):
        for j in range (len(X)):
            if X[j] <= 1.026 * Y[i]:
                Z[i][j] = -0.001

        continue
    return Z

def del_roman(Z, X, Y):
    for i in range(len(Y)):
        for j in range(len(X)):
            if 1.9 * Y[i] <= X[j]:
                Z[i][j] = -0.001
        continue
    return Z

# reading sample out of the file
def read_data(filename, line=None):
    with open(filename) as file:
        lines = file.readlines()
        y = parse_line(lines[3])
        x = []
        # making matrix for the input data
        matrix = []
        for i in range(4, len(lines)):
            numbers = parse_line(lines[i])
            x.append(numbers[0])
            if len(numbers) < 52:
                numbers.extend([0 for _ in range(77 - len(numbers))])
            zs = [numbers[j] for j in range(1, len(numbers))]
            matrix.append(zs)
        z = np.array(matrix).T
        return x, y, z

# getting samples out of files list
def get_data(files_list, dir):
    z_samples = []
    for filename in files_list:
        X, Y, z_matrix = read_data(dir + filename + data.file_type)
        # smoothing the input samples and making common samples list
        z_samples.append(np.ravel(del_roman(del_ryley(z_matrix, X, Y), X, Y)))
        # for getting raw samples
        #z_samples.append(np.ravel(z_matrix))
    return X, Y, z_samples

# PCA decomposition of the samples represented with res_shape shape
def get_pca_res(samples, res_shape, region):
    pca = PCA()
    # making long list of the input data
    pca_data = pca.fit_transform(np.array(samples).T.tolist())
    # drawing histogram
    per_var = np.round(pca.explained_variance_ratio_ * 100, decimals=1)
    labels = ['pc_' + str(x) for x in range(1, len(per_var) + 1)]
    # plot.bar(x=range(1, 11), height=per_var[0:10], tick_label=labels[0:10])
    plot.bar(labels, per_var)
    plot.title(region)
    # getting principal components
    pca_df = pd.DataFrame(pca_data, columns=labels)
    # xy_grids = np.meshgrid(x, y)
    comp1_grid = np.reshape(pca_df.pc_1.to_numpy(), res_shape)
    comp2_grid = np.reshape(pca_df.pc_2.to_numpy(), res_shape)
    return comp1_grid, comp2_grid

# use to draw 3 dimensional graph
def draw_3d_plot(x_mesh, y_mesh, z_mesh, name):
    fig = plot.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.plot_surface(x_mesh, y_mesh, z_mesh, cmap='inferno')
    ax.set_xlabel('Emission, nm.')
    ax.set_ylabel('Excitation, nm.')
    ax.set_zlabel('Intencity')
    ax.set_xlabel(name)

# use to draw contour graph
def draw_contour(x_mesh, y_mesh, z_mesh, name):
    fig = plot.figure()
    ax = fig.add_subplot()
    cs = ax.contourf(x_mesh, y_mesh, z_mesh, cmap='inferno', levels=60)
    add_rects(ax)
    ax.set_xlabel('Emission, nm.')
    ax.set_ylabel('Excitation, nm.')
    ax.set_xlabel(name)
    plot.colorbar(cs)

if __name__ == '__main__':
    path = data.path_Kivu
    files = data.files_Kivu

    # listing all .txt files in the directory
    files_list = []
    for elem in files:
        files_list.append(elem)
    # files_list = filter(lambda str: str.endswith('.txt'), os.listdir('Permafrost'))
    # getting data from them
    x, y, z_samples = get_data(files_list, path)
    # getting x and y grids for graphics
    xy_grids = np.meshgrid(x, y)
    # getting 1'st and 2'nd principal components
    comp1_grid, comp2_grid = get_pca_res(z_samples, xy_grids[0].shape, path)
    # making graphs
    print(get_local_max(comp1_grid, 200, x, y))
    draw_3d_plot(xy_grids[0], xy_grids[1], comp1_grid, "1 comp")
    draw_contour(xy_grids[0], xy_grids[1], comp1_grid, "1 comp")
    print('get_local_max(comp2_grid)')
    print(get_local_max(comp2_grid, 50, x, y))
    draw_3d_plot(xy_grids[0], xy_grids[1], comp2_grid, "2 comp")
    draw_contour(xy_grids[0], xy_grids[1], comp2_grid, "2 comp")
    plot.show()


from flask import Flask, render_template, request, jsonify, redirect
from database import signup_to_db, login_from_db, mobile_bill_db, elec_bill_db, gas_bill_db,broadband_bill_db, dth_bill_db, update_elec_last_bill_date, update_mobile_last_bill_date, update_gas_last_bill_date, update_broadband_last_bill_date, update_dth_last_bill_date
import random
from datetime import date, datetime

app= Flask(__name__)

@app.route("/")
def home():
  return render_template('home.html')

@app.route("/signup")
def signup():
  return render_template('signup.html')

@app.route("/signup/apply", methods=['post'])
def signup_done():
  data= request.form
  # return jsonify(data)
  signup_to_db(data)
  return render_template('signup_success.html')

@app.route("/login")
def login():
  return render_template('login.html')

@app.route("/login/success", methods=['post'])
def login_success():
  data= request.form
  is_user=login_from_db(data)

  if not is_user:
    return render_template('login_fail.html')

  return render_template('login_success.html')
  # return jsonify(data)

@app.route("/login/dashboard")
def dashboard():
  return render_template('dashboard.html')

@app.route("/login/elec")
def elec():
  return render_template('elec.html')

@app.route("/login/gas")
def gas():
  return render_template('gas.html')

@app.route("/login/broadband")
def broadband():
  return render_template('broadband.html')

@app.route("/login/dth")
def dth():
  return render_template('dth.html')

# @app.route("/login/mobile")
# def mobile():
#   return render_template('mobile.html')

@app.route("/login/mobile")
def water():
  return render_template('mobile.html')

@app.route("/elec/first_time")
def elec_first_time():
  today=date.today()
  id=random.randrange(2000000,3000000)
  return render_template('elec_first_time.html',today=today,id=id)

@app.route("/gas/first_time")
def gas_first_time():
  today=date.today()
  id=random.randrange(3000000,4000000)
  return render_template('gas_first_time.html',today=today,id=id)

@app.route("/mobile/first_time")
def mobile_first_time():
  today=date.today()
  id=random.randrange(1000000,2000000)
  return render_template('mobile_first_time.html',today=today,id=id)

@app.route("/broadband/first_time")
def broadband_first_time():
  today=date.today()
  id=random.randrange(4000000,5000000)
  return render_template('broadband_first_time.html',today=today,id=id)

@app.route("/dth/first_time")
def dth_first_time():
  today=date.today()
  id=random.randrange(5000000,6000000)
  return render_template('dth_first_time.html',today=today,id=id)

@app.route("/elec/re")
def elec_re():
  today=date.today()
  return render_template("elec_re.html", today=today)

@app.route("/gas/re")
def gas_re():
  today=date.today()
  return render_template("gas_re.html", today=today)

@app.route("/mobile/re")
def mobile_re():
  today=date.today()
  return render_template("mobile_re.html", today=today)

@app.route("/broadband/re")
def broadband_re():
  today=date.today()
  return render_template("broadband_re.html", today=today)

@app.route("/dth/re")
def dth_re():
  today=date.today()
  return render_template("dth_re.html", today=today)

@app.route("/elec/bill", methods=['post'])
def elec_bill():
  data= request.form
  # return jsonify(data)
  db_data=elec_bill_db(data)

  units=random.randrange(90,170)
  price_per_unit=10.73
  bill=units * price_per_unit

  curr_date = db_data['curr_date'].strftime('%Y-%m-%d')
  last_bill_date = db_data['last_bill_date'].strftime('%Y-%m-%d')

  if (datetime.strptime(curr_date, "%Y-%m-%d")-datetime.strptime(last_bill_date, "%Y-%m-%d")).days >30:
    update_elec_last_bill_date(data)
    return render_template('pay_elec_bill.html',db_data=db_data,units=units, price_per_unit=price_per_unit , bill=bill)

  else:
    return render_template("no_bill.html")

@app.route("/gas/bill", methods=['post'])
def gas_bill():
  data= request.form
  # return jsonify(data)
  db_data=gas_bill_db(data)

  units=random.randrange(20,37)
  price_per_unit=25
  bill=units * price_per_unit

  curr_date = db_data['curr_date'].strftime('%Y-%m-%d')
  last_bill_date = db_data['last_bill_date'].strftime('%Y-%m-%d')

  if (datetime.strptime(curr_date, "%Y-%m-%d")-datetime.strptime(last_bill_date, "%Y-%m-%d")).days >30:
    update_gas_last_bill_date(data)
    return render_template('pay_gas_bill.html',db_data=db_data,units=units, price_per_unit=price_per_unit , bill=bill)

  else:
    return render_template("no_bill.html")

@app.route("/mobile/bill", methods=['post'])
def mobile_bill():
  data= request.form
  # return jsonify(data)
  db_data=mobile_bill_db(data)

  curr_date = db_data['curr_date'].strftime('%Y-%m-%d')
  last_bill_date = db_data['last_bill_date'].strftime('%Y-%m-%d')

  if (datetime.strptime(curr_date, "%Y-%m-%d")-datetime.strptime(last_bill_date, "%Y-%m-%d")).days >30:
    update_mobile_last_bill_date(data)
    return render_template('pay_mobile_bill.html',db_data=db_data)

  else:
    return render_template("no_bill.html")

@app.route("/broadband/bill", methods=['post'])
def broadband_bill():
  data= request.form
  # return jsonify(data)
  db_data=broadband_bill_db(data)

  curr_date = db_data['curr_date'].strftime('%Y-%m-%d')
  last_bill_date = db_data['last_bill_date'].strftime('%Y-%m-%d')

  if (datetime.strptime(curr_date, "%Y-%m-%d")-datetime.strptime(last_bill_date, "%Y-%m-%d")).days >30:
    update_broadband_last_bill_date(data)
    return render_template('pay_broadband_bill.html',db_data=db_data)

  else:
    return render_template("no_bill.html")

@app.route("/dth/bill", methods=['post'])
def dth_bill():
  data= request.form
  # return jsonify(data)
  db_data=dth_bill_db(data)

  curr_date = db_data['curr_date'].strftime('%Y-%m-%d')
  last_bill_date = db_data['last_bill_date'].strftime('%Y-%m-%d')

  if (datetime.strptime(curr_date, "%Y-%m-%d")-datetime.strptime(last_bill_date, "%Y-%m-%d")).days >30:
    update_dth_last_bill_date(data)
    return render_template('pay_dth_bill.html',db_data=db_data)

  else:
    return render_template("no_bill.html")

@app.route("/login/bill_paid")
def bill_paid():
  return render_template('bill_paid.html')

@app.route("/contact")
def contact():
  return render_template('contact.html')

if __name__ =='__main__':
  app.run(host='0.0.0.0', debug=True)


# -*- coding: utf-8 -*-
import scrapy
from bs4 import BeautifulSoup

class BbcworldSpider(scrapy.Spider):
    name = 'bbcworld'
    #allowed_domains = ['http://www.baidu.com']
    start_urls = ['http://www.bbc.com/news/world']

    def parse(self, response):     #找头条
        bbc_st = response.text
        bbc_st_soup = BeautifulSoup(bbc_st, 'lxml')
        url = 'http://www.bbc.com' + bbc_st_soup.find('a',{'class':'title-link'}).get('href')
        yield scrapy.Request(url, callback=self.parseAgain)

    def parseAgain(self, response):    #找文章
        bbc_f = response.text
        bbc_soup = BeautifulSoup(bbc_f, 'lxml')
        essay = bbc_soup.head.title.string + '\n'     #标题
        for i in bbc_soup.body.find('p',{"class":"story-body__introduction"}).previous_sibling.next_siblings:
            if i.name in ['p'] :
                essay = self.findText(essay,i)
            elif i.name in ['h1','h2'] :
                essay += '\n'
                essay = self.findText(essay, i)
            elif i.name == 'hr':
                if i.next_sibling.name in ['h1','h2']:
                    essay += '\n'
                    essay = self.findText(essay, i)
                else:
                    break
            elif i.name in ['ul','li']:
                essay = self.findTextList(essay,i)
            else:
                continue
        with open('bbcworld.txt', 'w',encoding='utf-8',errors='ignore') as f:
            f.write(essay)
        self.log('Saved file bbcworld.')

    def findText(self,in_txt,in_soup):
        try:
            in_txt += '\n' + in_soup
        except:
            for x in in_soup.descendants:
                try:
                    in_txt += x
                except:
                    continue
            in_txt += '\n'
        return in_txt

    def findTextList(self,in_txt,in_soup):
        try:
            in_txt += '\n' + in_soup
        except:
            for x in in_soup.descendants:
                try:
                    in_txt += x
                except:
                    continue
        in_txt += '\n'
        return in_txt


import sqlite3
import datetime
import logging
import config

def db_connect():
	try:
		con = sqlite3.connect(config.FILE_DB)
		return con
	except sqlite3.Error as e:
		logging.critical(e)

def create_tables():
	con = db_connect()
	cur = con.cursor()

	cur.execute('''
		CREATE table if not exists referendums(
			chat_id integer,
			msg_id integer,
			title text,
			rfr_type integer,
			status integer,
			game_cost integer,
			max_players integer,
			user_id integer,
			user_name text,
			datum text,
			primary key(chat_id, msg_id))
	''')

	cur.execute('''
		CREATE table if not exists rfr_buttons(
			chat_id integer,
			msg_id integer,
			button_id integer,
			button_text text,
			primary key(chat_id, msg_id, button_id))
	''')

	cur.execute('''
		CREATE table if not exists rfr_log(
			chat_id integer,
			msg_id integer,
			button_id integer,
			user_id integer,
			user_name text,
			datum text,
			button_status integer,
			primary key(chat_id, msg_id, button_id, user_id, user_name, datum))
	''')

	cur.execute('''
		CREATE table if not exists regular_players(
			chat_id integer,
			user_id integer,
			user_name text,
			player_type integer,
			primary key(chat_id, user_id))
	''')

	cur.execute('''
		CREATE table if not exists timers(
			chat_id integer,
			msg_id integer,
			datum text,
			period integer,
			status integer,
			primary key(chat_id, msg_id))
	''')

	con.commit()
	con.close()

def exec_sql(sql, vals):
	con = db_connect()
	cur = con.cursor()

	try:
		rows = cur.executemany(sql, vals).fetchall()
		con.commit()
		con.close()
		return rows
	except sqlite3.Error as e:
		logging.warning(e)

def create_referendum_db(chat_id, msg_id, user_id, user_name, rfr_type, args):
	game_cost = args[0]
	max_players = args[1]
	title = args[2]
	buttons = args[3:]

	sql = '''
		INSERT into referendums(chat_id, msg_id, title, rfr_type, status, game_cost, max_players, user_id, user_name, datum)
			values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
	'''
	row = [(chat_id, msg_id, title, rfr_type, 1, game_cost, max_players, user_id, user_name, datetime.datetime.now())]
	exec_sql(sql, row)

	sql = '''
		INSERT into rfr_buttons(chat_id, msg_id, button_id, button_text)
			values(?, ?, ?, ?)
	'''
	rows = []
	for button_id in range(1, len(buttons) + 1):
		rows.append((chat_id, msg_id, button_id, buttons[button_id - 1]))
	exec_sql(sql, rows)

def get_referendums_by_user_id_db(chat_id, user_id, status = -1):
	referendums = []

	con = db_connect()
	con.row_factory = sqlite3.Row
	cur = con.cursor()

	if status == -1:
		sql = '''
			SELECT *
				from referendums
				where
					referendums.chat_id = {} and
					referendums.user_id = {}
		'''.format(chat_id, user_id)
	else:
		sql = '''
			SELECT *
				from referendums
				where
					referendums.chat_id = {} and
					referendums.status = {} and
					referendums.user_id = {}
		'''.format(chat_id, status, user_id)

	rows = cur.execute(sql).fetchall()
	con.close()

	for row in rows:
		referendum = {}

		referendum['msg_id'] = row['msg_id']
		referendum['title'] = row['title']
		referendum['rfr_type'] = row['rfr_type']
		referendum['status'] = row['status']
		referendum['game_cost'] = row['game_cost']
		referendum['max_players'] = row['max_players']
		referendum['user_id'] = row['user_id']
		referendum['user_name'] = row['user_name']
		referendum['datum'] = row['datum']

		referendums.append(referendum)

	return referendums

def get_referendum_db(chat_id, msg_id):
	referendum = {}

	con = db_connect()
	con.row_factory = sqlite3.Row
	cur = con.cursor()

	sql = '''
		SELECT *
			from referendums
			where
				referendums.chat_id = {} and
				referendums.msg_id = {}
	'''.format(chat_id, msg_id)

	row = cur.execute(sql).fetchone()
	con.close()

	if row:
		referendum['title'] = row['title']
		referendum['rfr_type'] = row['rfr_type']
		referendum['status'] = row['status']
		referendum['game_cost'] = row['game_cost']
		referendum['max_players'] = row['max_players']
		referendum['user_id'] = row['user_id']

	return referendum

def update_referendum_db(chat_id, args):
	msg_id = int(args[0])

	if len(args) >= 2:
		game_cost = int(args[1])

		sql = '''
			UPDATE referendums
				set game_cost = ?
				where
					chat_id = ? and
					msg_id = ?
		'''
		row = [(game_cost, chat_id, msg_id)]
		exec_sql(sql, row)

	if len(args) >= 3:
		max_players = int(args[2])

		sql = '''
			UPDATE referendums
				set max_players = ?
				where
					chat_id = ? and
					msg_id = ?
		'''
		row = [(max_players, chat_id, msg_id)]
		exec_sql(sql, row)

	if len(args) >= 4:
		title = args[3]

		sql = '''
			UPDATE referendums
				set title = ?
				where
					chat_id = ? and
					msg_id = ?
		'''
		row = [(title, chat_id, msg_id)]
		exec_sql(sql, row)

	if len(args) >= 5:
		buttons = args[4:]
		button_id = 1

		for button in buttons:
			sql = '''
				UPDATE rfr_buttons
					set button_text = ?
					where
						chat_id = ? and
						msg_id = ? and
						button_id = ?
			'''
			row = [(button, chat_id, msg_id, button_id)]
			exec_sql(sql, row)

			button_id += 1

def set_referendum_status_db(chat_id, msg_id, status):
	sql = '''
		UPDATE referendums
			SET status = ?
			WHERE
				chat_id = ? and
				msg_id = ?
	'''
	row = [(status, chat_id, msg_id)]
	exec_sql(sql, row)

def get_last_button_add(chat_id, msg_id, user_id, button_id):
	last_button_add = {}

	con = db_connect()
	con.row_factory = sqlite3.Row
	cur = con.cursor()

	sql = '''
		SELECT user_id, user_name, button_status, max(datum) as datum
			from rfr_log
			where
				chat_id = {} and
				msg_id = {} and
				button_id = {} and
				user_id = {} and
				button_status = 1
			group by user_id, user_name

	'''.format(chat_id, msg_id, config.BUTTON_ID_ADD, user_id)

	row = cur.execute(sql).fetchone()
	con.close()

	if row:
		last_button_add['user_id'] = row['user_id']
		last_button_add['user_name'] = row['user_name']
		last_button_add['button_status'] = row['button_status']
		last_button_add['datum'] = row['datum']

	return last_button_add

def set_vote_db(chat_id, msg_id, user_id, user_name, button_id):
	action = ''
	button_old = 0
	datum = datetime.datetime.now()
	referendum = get_referendum_db(chat_id, msg_id)
	votes = get_votes_db(chat_id, msg_id)

	if referendum['rfr_type'] in (config.RFR_GAME, config.RFR_GAME2):
		if button_id == config.BUTTON_ID_ADD:
			sql = '''
				INSERT into rfr_log(chat_id, msg_id, button_id, user_id, user_name, datum, button_status)
					values(?, ?, ?, ?, ?, ?, ?)
			'''
			row = [(chat_id, msg_id, button_id, user_id, user_name, datum, 1)]
			exec_sql(sql, row)

			action = 'added friend'
		elif button_id == config.BUTTON_ID_DEL:
			friends = get_friends_db(chat_id, msg_id)

			if user_id in friends:
				sql = '''
					INSERT into rfr_log(chat_id, msg_id, button_id, user_id, user_name, datum, button_status)
						values(?, ?, ?, ?, ?, ?, ?)
				'''
				row = [(chat_id, msg_id, button_id, user_id, user_name, datum, 0)]
				exec_sql(sql, row)

				last_button_add = get_last_button_add(chat_id, msg_id, user_id, config.BUTTON_ID_ADD)
				if last_button_add:
					sql = '''
						UPDATE rfr_log
							SET button_status = ?
							WHERE
								chat_id = ? and
								msg_id = ? and
								button_id = ? and
								user_id = ? and
								user_name = ? and
								datum = ?
					'''
					row = [(0, chat_id, msg_id, config.BUTTON_ID_ADD, user_id, user_name, last_button_add['datum'])]
					exec_sql(sql, row)

				action = 'removed friend'
			else:
				action = 'tried to remove 0 friends'
		elif button_id == config.BUTTON_ID_OPT:
			sql = '''
				INSERT into rfr_log(chat_id, msg_id, button_id, user_id, user_name, datum, button_status)
					values(?, ?, ?, ?, ?, ?, ?)
			'''

			for usr in votes[button_id]['players']:
				if usr['user_id'] == user_id:
					row = [(chat_id, msg_id, button_id, user_id, user_name, datum, 0)]
					action = 'unvoted'
				else:
					action = ''
			if action == '':
				row = [(chat_id, msg_id, button_id, user_id, user_name, datum, 1)]
				action = 'voted'

			exec_sql(sql, row)

		else:
			for btn_id in (config.BUTTON_ID_YES, 2, 3):
				for usr in votes[btn_id]['players'] + votes[btn_id]['queue']:
					if usr['user_id'] == user_id:
						sql = '''
							INSERT into rfr_log(chat_id, msg_id, button_id, user_id, user_name, datum, button_status)
								values(?, ?, ?, ?, ?, ?, ?)
						'''
						row = [(chat_id, msg_id, btn_id, user_id, user_name, datum, 0)]
						exec_sql(sql, row)

						button_old = btn_id
						action = 'unvoted'

			if button_old != button_id:
				sql = '''
					INSERT into rfr_log(chat_id, msg_id, button_id, user_id, user_name, datum, button_status)
						values(?, ?, ?, ?, ?, ?, ?)
				'''
				row = [(chat_id, msg_id, button_id, user_id, user_name, datum, 1)]
				exec_sql(sql, row)

				if action:
					action = 'revoted'
				else:
					action = 'voted'
	elif referendum['rfr_type'] == config.RFR_SINGLE:
		for btn_id in votes:
			for usr in votes[btn_id]['players'] + votes[btn_id]['queue']:
				if usr['user_id'] == user_id:
					sql = '''
						INSERT into rfr_log(chat_id, msg_id, button_id, user_id, user_name, datum, button_status)
							values(?, ?, ?, ?, ?, ?, ?)
					'''
					row = [(chat_id, msg_id, btn_id, user_id, user_name, datum, 0)]
					exec_sql(sql, row)

					button_old = btn_id
					action = 'unvoted'

		if button_old != button_id:
			sql = '''
				INSERT into rfr_log(chat_id, msg_id, button_id, user_id, user_name, datum, button_status)
					values(?, ?, ?, ?, ?, ?, ?)
			'''
			row = [(chat_id, msg_id, button_id, user_id, user_name, datum, 1)]
			exec_sql(sql, row)

			if action:
				action = 'revoted'
			else:
				action = 'voted'
	elif referendum['rfr_type'] == config.RFR_MULTI:
		for btn_id in votes:
			for usr in votes[btn_id]['players'] + votes[btn_id]['queue']:
				if usr['user_id'] == user_id and btn_id == button_id:
					sql = '''
						INSERT into rfr_log(chat_id, msg_id, button_id, user_id, user_name, datum, button_status)
							values(?, ?, ?, ?, ?, ?, ?)
					'''
					row = [(chat_id, msg_id, btn_id, user_id, user_name, datum, 0)]
					exec_sql(sql, row)

					button_old = btn_id
					action = 'unvoted'

		if action == '':
			sql = '''
				INSERT into rfr_log(chat_id, msg_id, button_id, user_id, user_name, datum, button_status)
					values(?, ?, ?, ?, ?, ?, ?)
			'''
			row = [(chat_id, msg_id, button_id, user_id, user_name, datum, 1)]
			exec_sql(sql, row)

			action = 'voted'

	return action

def get_votes_db(chat_id, msg_id):
	referendum_log = []

	referendum = get_referendum_db(chat_id, msg_id)
	buttons = get_buttons_db(chat_id, msg_id)

	con = db_connect()
	con.row_factory = sqlite3.Row
	cur = con.cursor()

	sql = '''
		SELECT rfr_buttons.button_id, user_id, user_name, max(datum) as datum, button_status
			from rfr_buttons
			left outer join rfr_log
			on
				rfr_buttons.chat_id = rfr_log.chat_id and
				rfr_buttons.msg_id = rfr_log.msg_id and
				rfr_buttons.button_id = rfr_log.button_id
			where
				rfr_buttons.chat_id = {} and
				rfr_buttons.msg_id = {}
			group by
				rfr_buttons.button_id, user_id, user_name
			order by
				rfr_buttons.button_id, datum
	'''.format(chat_id, msg_id)
	rows = cur.execute(sql).fetchall()
	con.close()

	for row in rows:
		if(row['button_status']):
			referendum_log.append({'button_id': row['button_id'], 'user_id': row['user_id'], 'user_name': row['user_name'], 'datum': row['datum']})

	referendum_log = sorted(referendum_log, key = lambda x: x['datum'])
	players_queue = get_players_queue(referendum_log, buttons, referendum['rfr_type'], referendum['max_players'])

	return players_queue

def get_buttons_db(chat_id, msg_id):
	buttons = {}

	con = db_connect()
	con.row_factory = sqlite3.Row
	cur = con.cursor()

	sql = '''
		SELECT button_id, button_text
			from rfr_buttons
			where
				chat_id = {} and
				msg_id = {}
	'''.format(chat_id, msg_id)
	rows = cur.execute(sql).fetchall()
	con.close()

	for row in rows:
		buttons[row['button_id']] = {'button_text': row['button_text']}

	return buttons

def drop_tables():
	con = db_connect()
	cur = con.cursor()

	cur.execute('drop table if exists referendums')
	cur.execute('drop table if exists rfr_buttons')
	cur.execute('drop table if exists rfr_log')
	cur.execute('drop table if exists regular_players')

	con.commit()
	con.close()

def extend_table():
	con = db_connect()
	cur = con.cursor()

	cur.execute('drop table if exists _referendums_old')
	cur.execute('ALTER table referendums rename to _referendums_old')
	cur.execute('''
		CREATE table referendums(
			chat_id integer,
			msg_id integer,
			title text,
			rfr_type integer default 0,
			status integer,
			game_cost integer default 0,
			max_players integer,
			user_id integer,
			user_name text,
			datum text,
			primary key(chat_id, msg_id))''')

	cur.execute('''
		INSERT into referendums(
					chat_id, msg_id, title, status, game_cost, max_players, user_id, user_name, datum)
			select chat_id, msg_id, title, status, game_cost, max_players, user_id, user_name, datum
			from _referendums_old
	''')
	cur.execute('drop table _referendums_old')

	con.commit()
	con.close()

def drop_table_column():
	con = db_connect()
	cur = con.cursor()

	cur.execute('drop table if exists _rfr_buttons_old')
	cur.execute('ALTER table rfr_buttons rename to _rfr_buttons_old')
	cur.execute('''
		CREATE table rfr_buttons(
			chat_id integer,
			msg_id integer,
			button_id integer,
			button_text text,
			primary key(chat_id, msg_id, button_id))
	''')

	cur.execute('''
		INSERT into rfr_buttons(
					chat_id, msg_id, button_id, button_text)
			select chat_id, msg_id, button_id, button_text
			from _rfr_buttons_old''')
	cur.execute('drop table _rfr_buttons_old')

	con.commit()
	con.close()

def select_all(tab):
	con = db_connect()
	cur = con.cursor()

	rows = []
	for row in cur.execute(f'select * from {tab}'):
		rows.append(row)
	con.close()

	return rows

def print_tabs(*args):
	if(args):
		for arg in args:
			print(arg)
			rows = select_all(arg)
			for row in rows:
				print(row)
	else:
		print('referendums')
		rows = select_all('referendums')
		for row in rows:
			print(row)

		print('rfr_buttons')
		rows = select_all('rfr_buttons')
		for row in rows:
			print(row)

		print('rfr_log')
		rows = select_all('rfr_log')
		for row in rows:
			print(row)

		print('regular_players')
		rows = select_all('regular_players')
		for row in rows:
			print(row)

def is_free_slots(players_queue, buttons, max_num):
	busy_slots = 0

	for button in buttons:
		if button == 1:
			busy_slots += len(players_queue[button]['players'])

	if busy_slots + 1 <= max_num:
		return True
	else:
		return False

def get_players_queue(referendum_log, buttons, rfr_type, max_num):
	players_queue = {}

	for button in buttons:
		players_queue[button] = {'players': [], 'queue': []}

	for r in referendum_log:
		button_id = r['button_id']

		if rfr_type in (config.RFR_GAME, config.RFR_GAME2) and button_id == 1 and max_num > 0:
			if is_free_slots(players_queue, buttons, max_num):
				players_queue[r['button_id']]['players'].append(r)
			else:
				players_queue[r['button_id']]['queue'].append(r)
		elif rfr_type in (config.RFR_GAME, config.RFR_GAME2) and button_id in (4, 5):
			pass
		else:
			players_queue[r['button_id']]['players'].append(r)

	return players_queue

def check_user_id(chat_id, msg_id, user_id):
	referendum = get_referendum_db(chat_id, msg_id)

	if user_id == referendum['user_id']:
		return True
	else:
		return False

def check_msg_id(chat_id, msg_id):
	referendum = get_referendum_db(chat_id, msg_id)

	if referendum:
		return True
	else:
		return False

def get_regular_players_db(chat_id , player_type = -1):
	players = []

	con = db_connect()
	con.row_factory = sqlite3.Row
	cur = con.cursor()

	if player_type == -1:
		sql = '''
			SELECT *
				from regular_players
				where chat_id = {}
		'''.format(chat_id)
	else:
		sql = '''
			SELECT *
				from regular_players
				where chat_id = {} and
					player_type = {}
		'''.format(chat_id, player_type)

	rows = cur.execute(sql).fetchall()
	con.close()

	for row in rows:
		players.append({'chat_id': row['chat_id'], 'user_id': row['user_id'], 'user_name': row['user_name'], 'player_type': row['player_type']})

	return players

def is_regular_players_used_db(chat_id):
	con = db_connect()
	con.row_factory = sqlite3.Row
	cur = con.cursor()

	sql = '''
		SELECT *
			from regular_players
			where
				chat_id = {} and
				player_type = 1
	'''.format(chat_id)

	row = cur.execute(sql).fetchone()

	if row:
		return True
	else:
		return False

def get_regular_player_db(chat_id, user_id):
	player = {}

	con = db_connect()
	con.row_factory = sqlite3.Row
	cur = con.cursor()

	sql = '''
		SELECT *
			from regular_players
			where
				chat_id = {} and
				user_id = {}
	'''.format(chat_id, user_id)

	row = cur.execute(sql).fetchone()

	if row:
		player['chat_id'] = row['chat_id']
		player['user_id'] = row['user_id']
		player['user_name'] = row['user_name']
		player['player_type'] = row['player_type']

	return player

def set_regular_player_db(chat_id, user_id, user_name, player_type):
	sql = '''
		INSERT or replace
			into regular_players(chat_id, user_id, user_name, player_type)
			values(?, ?, ?, ?)
	'''
	row = [(chat_id, user_id, user_name, player_type)]

	exec_sql(sql, row)

def del_regular_player_db(chat_id, user_id):
	sql = '''
		DELETE
			from regular_players
			where
				chat_id = {} and
				user_id = {}
	'''.format(chat_id, user_id)

	con = db_connect()
	cur = con.cursor()
	cur.execute(sql)
	con.commit()
	con.close()

def is_regular_player(chat_id, user_id):
	player = get_regular_player_db(chat_id, user_id)

	if player:
		return player['player_type']
	else:
		return 0

def get_friends_db(chat_id, msg_id):
	friends = {}

	con = db_connect()
	con.row_factory = sqlite3.Row
	cur = con.cursor()

	sql = '''
		SELECT user_id, user_name, sum(button_status) as friends, min(datum) as datum
			from rfr_log
			where
				chat_id = {} and
				msg_id = {} and
				button_id = 4 and
				button_status = 1
			group by user_id, user_name
			order by datum
	'''.format(chat_id, msg_id)
	rows = cur.execute(sql).fetchall()
	con.close()

	for row in rows:
		if row['friends'] > 0:
			friends[row['user_id']] = {'user_name': row['user_name'], 'friends': row['friends'], 'datum': row['datum']}

	return friends

def add_button(chat_id, msg_id, button_text):
	con = db_connect()
	con.row_factory = sqlite3.Row
	cur = con.cursor()

	sql = '''
		SELECT max(button_id) as button_id
			from rfr_buttons
			where
				chat_id = {} and
				msg_id = {}
	'''.format(chat_id, msg_id)

	row = cur.execute(sql).fetchone()
	con.close()

	if row:
		button_id = row['button_id'] + 1

		sql = '''
			INSERT into rfr_buttons(chat_id, msg_id, button_id, button_text)
				values(?, ?, ?, ?)
		'''
		row = [(chat_id, msg_id, button_id, button_text)]
		exec_sql(sql, row)

def get_silent_members_db(chat_id, msg_id):
	chat_members = []
	active_members = []
	silent_members = []

	con = db_connect()
	con.row_factory = sqlite3.Row
	cur = con.cursor()

	sql = '''
		SELECT user_id
			from regular_players
			where chat_id = {}
	'''.format(chat_id)
	rows = cur.execute(sql).fetchall()
	for row in rows:
		chat_members.append({'user_id': row['user_id']})

	buttons = get_buttons_db(chat_id, msg_id)
	votes = get_votes_db(chat_id, msg_id)

	for button_id in buttons:
		for usr in votes[button_id]['players']:
			active_members.append({'user_id': usr['user_id']})
		for usr in votes[button_id]['queue']:
			active_members.append({'user_id': usr['user_id']})

	silent_members = [p for p in chat_members if p not in active_members]

	return silent_members


from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from bs4 import BeautifulSoup
import jsonlines

class someSpider(CrawlSpider):
    name = 'getquestions'
    allowed_domains = ['datascience.stackexchange.com']
    start_urls = ['https://datascience.stackexchange.com/questions/6107/what-are-deconvolutional-layers']
    rules = (Rule (LinkExtractor(allow="/questions/*"), callback="parse_obj", follow=True),)

    def parse_obj(self,response):
        with jsonlines.open('data.jsonl', mode='a') as writer:

            url = response.url
            question_head = response.xpath('//div[@id="question-header"]/h1/a/text()').get()
            question_body = ' '.join(response.xpath('//div[@class="s-prose js-post-body"]/p/text()').getall())

            soup = BeautifulSoup(response.text, 'lxml')
            html = [x.text for x in soup.find_all("div",attrs={"class":"answer accepted-answer"})]
            html = ' '.join(html)
            html = " ".join(line.strip() for line in html.split("\n"))
            if html:
                answer_body = html
            else:
                html = [x.text for x in soup.find_all("div",attrs={"class":"answer"})]
                html = ' '.join(html)
                html = " ".join(line.strip() for line in html.split("\n"))
                answer_body = html

            obj = {
                'url' : url,
                'question_head': question_head,
                'question_body': question_body,
                'answer_body': answer_body
            }
            writer.write(obj)


import pytest

def army_to_list(army):
    return [prop for troop in army for prop in troop]

@pytest.mark.asyncio
async def test_army_to_array(arrstructs):
    troop_vals = (10, 10, 100)
    troops = [arrstructs.Troop(*troop_vals)] * 3
    army = arrstructs.Army(*troops)
    tx = await arrstructs.army_to_array(army).invoke()
    assert tx.result.a == army_to_list(army)

@pytest.mark.asyncio
async def test_array_to_army(arrstructs):
    army = [10, 10, 100, 10, 10, 100, 10, 10, 100]
    tx = await arrstructs.array_to_army(army).invoke()
    assert army_to_list(tx.result.a) == army

@pytest.mark.asyncio
async def test_new_config(arrstructs):
    tx = await arrstructs.new_config(10, 20, 30).invoke()
    assert tx.result.config == arrstructs.Config(value1=10, value2=20, value3=30)


import bpy
from bpy.types import Region
from bpy.props import IntProperty

# Tag the given region for a redraw.
# Notably used for the message bus.
def require_region_redraw(region: bpy.types.Region):
    region.tag_redraw()

# Base class for the editor's regions.
class ViewLayerManagerRegion(Region):
    bl_space_type = 'VIEW_LAYER_MANAGER_EDITOR'

class ViewLayerManagerHeaderRegion(ViewLayerManagerRegion):
    bl_region_type = 'HEADER'

    def init(self, window_manager):
        Region.header_init(self)

class ViewLayerManagerUIRegion(ViewLayerManagerRegion):
    bl_region_type = 'UI'

    def init(self, window_manager):
        Region.panels_init(window_manager, self)

class ViewLayerManagerWindowRegion(ViewLayerManagerRegion):
    bl_region_type = 'WINDOW'

    def init(self, window_manager):
        Region.panels_init(window_manager, self)

        types = (bpy.types.ViewLayer, bpy.types.LayerCollection, bpy.types.Collection)

        for t in types:
            bpy.msgbus.subscribe_rna(
                key=t,
                owner=self,
                args=(self, ),
                notify=require_region_redraw,
            )

        bpy.msgbus.subscribe_rna(
            key=(bpy.types.Scene, "view_layers"),
            owner=self,
            args=(self, ),
            notify=require_region_redraw,
        )

# Update function for the query string.
# self is the editor.
def update_search_filter(self, context):
    self.search_filter_update()

class ViewLayerManagerSpace(bpy.types.Space):
    bl_idname = 'VIEW_LAYER_MANAGER_EDITOR'
    bl_label = 'View Layer Manager'
    bl_icon = 'RENDER_RESULT'

    search_filter: bpy.props.StringProperty(name="Search Filter", update=update_search_filter, options={'TEXTEDIT_UPDATE'})

    # Property for the active view layer in the UIList.
    active_view_layer: IntProperty()

    def search_filter_get(self, region):
        return self.search_filter

classes = (
    ViewLayerManagerSpace,
    ViewLayerManagerHeaderRegion,
    ViewLayerManagerUIRegion,
    ViewLayerManagerWindowRegion,
)


from sys import stdin
from collections import deque

N, M = map(int, stdin.readline().split())
check = [[[False for _ in range(2)] for _ in range(M)] for _ in range(N)]
Map = [list(stdin.readline().strip()) for _ in range(N)]

Que = deque([(0, 0, 0)])
check[0][0][0] = 1
check[0][0][1] = 1

dy = [0, 0, 1, -1]
dx = [1, -1, 0, 0]

output = -1

if N == 1 and M == 1:
    output = 1

while len(Que):
    cur_y, cur_x, chance = Que.popleft()
    for i in range(4):
        ny = cur_y + dy[i]
        nx = cur_x + dx[i]
        if 0 <= ny < N and 0 <= nx < M:
            if ny == N-1 and nx == M-1:
                output = check[cur_y][cur_x][chance] + 1
                Que.clear()
                break

            elif Map[ny][nx] == '0' and check[ny][nx][chance] is False:
                Que.append((ny, nx, chance))
                check[ny][nx][chance] = check[cur_y][cur_x][chance] + 1

            elif Map[ny][nx] == '1' and chance == 0:
                Que.append((ny, nx, 1))
                check[ny][nx][1] = check[cur_y][cur_x][chance] + 1

print(output)

6.연습/벽부수고이동하기.py


# coding=utf-8
import datetime
import logging
import os

import matplotlib
from flask import g

from ..base import Base
from ...db import get_db
from ...db.dao.component import Component
from ...db.dao.evaluation import Evaluation
from ...db.dao.evaluation_file import EvaluationFile
from ...db.dao.experimental_task import ExperimentalTask
from ...db.dao.project import Project
from ...db.dao.report import Report
from ...db.models import EvaluationFileModel
from ...extensions import get_file_client
from ...filesystem import get_fs

matplotlib.use('Agg')

logging.basicConfig(
    level=logging.ERROR,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

class ReportUpload(Base):

    menu_info_names = [{'name': '删除'},
                       {'name': '提交实验结果'}]

    evaluation_info_names = [{'name': 'log', 'type': 'text'}]

    def _get_evaluation_dir(self, evaluation_id):
        return 'evaluations/%s' % evaluation_id

    def _check_valid_params(self, logger, params=None):
        if not params:
            logger.exception('params is None')
            return False
        elif not params.get('parent_id'):
            logger.exception('params has no attribute name "parent_id"')
            return False
        elif not params.get('project_id'):
            logger.exception('params has no attribute name "project_id"')
            return False
        elif not params.get('parent_task_name'):
            logger.exception('params has no attribute name "parent_task_name"')
            return False
        else:
            return True

    def execute(self, evaluation_id=None, params=None, item_id=None):
        fs = get_fs()
        file_client = get_file_client()
        evaluation_dir = self._get_evaluation_dir(item_id)
        evaluation_output_dir = fs.join(evaluation_dir, 'outputs')
        if fs.isdir(fs.join(evaluation_dir, 'outputs')):
            fs.rmtree(fs.join(evaluation_dir, 'outputs'))
        fs.makedirs(evaluation_output_dir)
        formatter = logging.Formatter(
            '%(asctime)s - %(name)s - %(levelname)s: - %(message)s',
            datefmt='%Y-%m-%d %H:%M:%S')
        log_path = fs.abs_path(fs.join(evaluation_output_dir, 'evaluation.log'))
        fh = logging.FileHandler(log_path)
        fh.setFormatter(formatter)
        logger.addHandler(fh)
        success = self._check_valid_params(logger, params)
        content = params.get('content')
        file_key = params.get('file_key')
        parent_id = params['parent_id']
        parent_task_name = params['parent_task_name']
        project_id = params['project_id']
        project = Project().query(project_id=project_id)[0]
        experimental_task_id = project['experimental_task_id']
        from dmlab_app.task import get_task_method, get_customized_task_method
        task_method = get_task_method(parent_task_name)
        if not task_method:
            component = Component().query(component_name=parent_task_name, user_id=params['user_id'])[0]
            task_method = get_customized_task_method(component['component_type_id'])
        if success:
            experimental_task = ExperimentalTask().query(experimental_task_id=experimental_task_id)[0]
            score_field = experimental_task['score_field']
            print(params['user_id'])
            reports = Report().query(user_id=params['user_id'], experimental_task_id=experimental_task_id)
            time_value = (experimental_task['dead_line'] - datetime.datetime.now()) / (experimental_task['dead_line'] - experimental_task['start_time'])
            score = task_method.calc_score(score_field, parent_id, len(reports), time_value)
            score_content = task_method.get_score(parent_id)
            Report().create(user_id=params['user_id'], experimental_task_id=experimental_task_id, task_name=parent_task_name, data_id=parent_id, content=content, file_key=file_key, score_content=score_content, score=score)
        logger.removeHandler(fh)
        db = get_db()
        try:
            collection = file_client.get_collection()
            file_paths = list()

            for dirpath, dirnames, filenames in os.walk(fs.abs_path(evaluation_dir)):
                for filename in filenames:
                    file_path = os.path.join(dirpath, filename)
                    r_path = os.path.relpath(file_path, fs.abs_path(evaluation_dir))
                    with open(file_path, 'rb') as fin:
                        collection.add(fin.read())
                        file_paths.append(r_path)
            rets = file_client.upload_collection(collection)
            for idx, ret in enumerate(rets):
                evaluation_file = EvaluationFileModel(evaluation_id=evaluation_id,
                                                      file_path=file_paths[idx], file_key=ret.id, deleted=0)
                db.add(evaluation_file)
            collection.close()
            # os.remove(fs.abs_path(evaluation_dir))
            db.commit()
        except:
            db.rollback()
        finally:
            db.close()
        return success

    def get_evaluation_info_list(self, item_id, info_name, config=None, limit=None, offset=None):
        assert info_name in map(lambda info_type: info_type['name'], self.evaluation_info_names)
        if info_name == 'data':
            return self._get_evaluation_data(item_id, limit=limit, offset=offset)
        elif info_name == 'log':
            return self._get_evaluation_log(item_id, limit=limit, offset=offset)
        elif info_name == 'report':
            return self._get_evaluation_report(item_id, limit=limit, offset=offset)
        else:
            raise NotImplementedError

    def _get_evaluation_data(self, item_id, limit=None, offset=None):
        return None, None, None

    def _get_evaluation_log(self, item_id, limit=None, offset=None):
        file_client = get_file_client()
        evaluation = Evaluation().query(item_id=item_id)[0]
        evaluation_file = EvaluationFile().query(evaluation_id=evaluation['evaluation_id'],
                                                 file_path='outputs/evaluation.log')
        if evaluation_file:
            log_content = file_client.download(evaluation_file[0]['file_key'])
            logs = [{
                'id': 1,
                'name': 'evaluation.log',
                'type': 'text',
                'data': str(log_content, encoding='utf-8')
            }]
        else:
            logs = []
        count = len(logs)
        if limit is None:
            limit = len(logs)
        else:
            limit = int(limit)
        if offset is None:
            offset = 0
        else:
            offset = int(offset)
        return logs[offset:offset + limit], count, None

    def _get_evaluation_report(self, item_id, limit=None, offset=None):
        return None, None, None

    def get_score(self, item_id):
        return ''


from django.http import HttpResponse
from django.test import RequestFactory
from djblets.cache.backend import cache_memoize
from kgb import SpyAgency

from reviewboard.diffviewer.errors import UserVisibleError
from reviewboard.diffviewer.models import FileDiff
from reviewboard.diffviewer.renderers import DiffRenderer
from reviewboard.diffviewer.settings import DiffSettings
from reviewboard.testing import TestCase

class DiffRendererTests(SpyAgency, TestCase):
    """Unit tests for DiffRenderer."""

    def test_construction_with_invalid_chunks(self):
        """Testing DiffRenderer construction with invalid chunks"""
        diff_file = {
            'chunks': [{}],
            'filediff': None,
            'interfilediff': None,
            'force_interdiff': False,
            'chunks_loaded': True,
        }

        diff_settings = DiffSettings.create()

        renderer = DiffRenderer(diff_file, chunk_index=-1,
                                diff_settings=diff_settings)
        self.assertRaises(UserVisibleError,
                          lambda: renderer.render_to_string_uncached(None))

        renderer = DiffRenderer(diff_file, chunk_index=1,
                                diff_settings=diff_settings)
        self.assertRaises(UserVisibleError,
                          lambda: renderer.render_to_string_uncached(None))

    def test_construction_with_valid_chunks(self):
        """Testing DiffRenderer construction with valid chunks"""
        diff_file = {
            'chunks': [{}],
            'chunks_loaded': True,
        }

        # Should not assert.
        renderer = DiffRenderer(diff_file, chunk_index=0,
                                diff_settings=DiffSettings.create())
        self.spy_on(renderer.render_to_string, call_original=False)
        self.spy_on(renderer.make_context, call_original=False)

        renderer.render_to_string_uncached(None)
        self.assertEqual(renderer.num_chunks, 1)
        self.assertEqual(renderer.chunk_index, 0)

    def test_construction_with_setting_syntax_highlighting_true(self):
        """Testing DiffRenderer construction with
        DiffSettings.syntax_highlighting=True
        """
        diff_file = {
            'chunks': [{}],
            'chunks_loaded': True,
        }

        diff_settings = DiffSettings.create(syntax_highlighting=True)

        renderer = DiffRenderer(diff_file,
                                chunk_index=0,
                                diff_settings=diff_settings)
        self.assertTrue(renderer.highlighting)

    def test_construction_with_setting_syntax_highlighting_false(self):
        """Testing DiffRenderer construction with
        DiffSettings.syntax_highlighting=False
        """
        diff_file = {
            'chunks': [{}],
            'chunks_loaded': True,
        }

        diff_settings = DiffSettings.create(syntax_highlighting=False)

        renderer = DiffRenderer(diff_file,
                                chunk_index=0,
                                diff_settings=diff_settings)
        self.assertFalse(renderer.highlighting)

    def test_render_to_response(self):
        """Testing DiffRenderer.render_to_response"""
        diff_file = {
            'chunks': [{}]
        }

        renderer = DiffRenderer(diff_file, diff_settings=DiffSettings.create())
        self.spy_on(renderer.render_to_string,
                    call_fake=lambda self, request: 'Foo')

        request_factory = RequestFactory()
        request = request_factory.get('/')
        response = renderer.render_to_response(request)

        self.assertTrue(renderer.render_to_string.called)
        self.assertTrue(isinstance(response, HttpResponse))
        self.assertEqual(response.content, b'Foo')

    def test_render_to_string(self):
        """Testing DiffRenderer.render_to_string"""
        diff_file = {
            'chunks': [{}]
        }

        renderer = DiffRenderer(diff_file, diff_settings=DiffSettings.create())
        self.spy_on(renderer.render_to_string_uncached,
                    call_fake=lambda self, request: 'Foo')
        self.spy_on(renderer.make_cache_key,
                    call_fake=lambda self: 'my-cache-key')
        self.spy_on(cache_memoize)

        request_factory = RequestFactory()
        request = request_factory.get('/')
        response = renderer.render_to_response(request)

        self.assertEqual(response.content, b'Foo')
        self.assertTrue(renderer.render_to_string_uncached.called)
        self.assertTrue(renderer.make_cache_key.called)
        self.assertTrue(cache_memoize.spy.called)

    def test_render_to_string_uncached(self):
        """Testing DiffRenderer.render_to_string_uncached"""
        diff_file = {
            'chunks': [{}]
        }

        renderer = DiffRenderer(diff_file, lines_of_context=[5, 5],
                                diff_settings=DiffSettings.create())
        self.spy_on(renderer.render_to_string_uncached,
                    call_fake=lambda self, request: 'Foo')
        self.spy_on(renderer.make_cache_key,
                    call_fake=lambda self: 'my-cache-key')
        self.spy_on(cache_memoize)

        request_factory = RequestFactory()
        request = request_factory.get('/')
        response = renderer.render_to_response(request)

        self.assertEqual(response.content, b'Foo')
        self.assertTrue(renderer.render_to_string_uncached.called)
        self.assertFalse(renderer.make_cache_key.called)
        self.assertFalse(cache_memoize.spy.called)

    def test_make_context_with_chunk_index(self):
        """Testing DiffRenderer.make_context with chunk_index"""
        diff_file = {
            'newfile': True,
            'interfilediff': None,
            'filediff': FileDiff(),
            'chunks': [
                {
                    'lines': [],
                    'meta': {},
                    'change': 'insert',
                },
                {
                    # This is not how lines really look, but it's fine for
                    # current usage tests.
                    'lines': range(10),
                    'meta': {},
                    'change': 'replace',
                },
                {
                    'lines': [],
                    'meta': {},
                    'change': 'delete',
                }
            ],
        }

        renderer = DiffRenderer(diff_file, chunk_index=1,
                                diff_settings=DiffSettings.create())
        context = renderer.make_context()

        self.assertEqual(context['standalone'], True)
        self.assertEqual(context['file'], diff_file)
        self.assertEqual(len(diff_file['chunks']), 1)

        chunk = diff_file['chunks'][0]
        self.assertEqual(chunk['change'], 'replace')


import argparse
from typing import List

from dazu.components.engine import Engine
from dazu.config import DazuConfig

# noinspection PyProtectedMember
def add_subparser(
    subparsers: argparse._SubParsersAction, parents: List[argparse.ArgumentParser]
):
    train_parser = subparsers.add_parser(
        "train",
        parents=parents,
        conflict_handler="resolve",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        help="Starts train with Training Data with your configured pipeline.",
    )
    train_parser.set_defaults(func=train)

    # arguments.set_run_arguments(run_parser)

def train(config: DazuConfig, args: argparse.Namespace):
    engine = Engine(config)
    engine.train()


import urllib.request

from bs4 import BeautifulSoup

def request_list(page):
    url = 'http://movie.naver.com/movie/bi/mi/pointWriteFormList.nhn?code=127382&type=after&isActualPointWriteExecute=false&isMileageSubscriptionAlready=false&isMileageSubscriptionReject=false&page={page}'
    url = url.format(page=page)

    res = urllib.request.urlopen(url)
    html = res.read()

    return BeautifulSoup(html, 'html.parser')

def fetch_comm_detail(soup):
    result = []

    li_tags = soup.find('div', class_='score_result').find_all('li')

    for li in li_tags:
        score_reple = li.find('div', class_='score_reple')
        content = score_reple.find('p')

        if content.span:
            content.span.extract()

        if content.span:
            content.span.extract()

        content = content.get_text(strip=True)

        dt = score_reple.find('dl').find('dt')
        em_tags = dt.find_all('em')
        name = em_tags[0].find('span').get_text(strip=True)
        date = em_tags[1].get_text(strip=True)

        result.append({
            'content' : content,
            'name' : name,
            'date' : date
            })

    return result

for i in range(1, 4):
    soup = request_list(i)
    result = fetch_comm_detail(soup)
    print(result)