"""
This file contains the code to do the calculation.

Each resolver receives a model and outputs the calculated result, which can
be a model or just a matrix.
"""
from datetime import date, datetime
import pandas as pd
import warnings
import numpy as np
import os
import sys
import re
import matplotlib.pyplot as plt
from matplotlib.ticker import StrMethodFormatter
from matplotlib.pyplot import cm
from app.core.models import constant_file_config

warnings.filterwarnings('ignore')

class Header_data():
    """
    the object to store run info data which read from the head of the raw data
    """
    def __init__(self):
        self.CAL_num = ""
        self.Client_name = ""
        self.address_1 = ""
        self.address_2 = ""
        self.operator = ""
        self.serial = ""
        self.model = ""
        self.date = ""

class Processed_data():
    """
    the object to store data which read from the raw data
    """
    def __init__(self):
        self.df_before_mean = None
        self.df_mean = None
        self.df_after_mean = None

class result_data():
    """
    Using object to save result in order to avoid too much returned elements
    """
    def __init__(self):
        self.df_NK = None
        self.df_leakage = None
        self.highlight = []
        self.X = []
        self.df_otherConstant = None

class HeaderError(RuntimeError):
    def __init__(self, arg):
        self.args = arg

def calculator(client, lab):
    """
    This function will calculate one run of NK (calibration coefficient) and the leakage current values. Besides,
    the coordinate of the leakage value which need to be highlighted is needed so that the front-end will be easy to
    highlight the cell.

    :param client: the client raw data path
    :param lab   : the lab raw data path
    :return      : return a object of the result which includes NK dataframe, leakage value dataframe, the coordinate of
                   the leakage value which need to be highlighted (abs(value) > 0.2), the effective energy
                   dataframe (X), and the list of NK (Y). X and Y is used to plot on chart.
    """
    # extract data
    client_data, duplicate_num = extraction(client)
    lab_data, _ = extraction(lab)

    # order the lab beams to make sure that the further calculation will not have issue
    cats = client_data.df_mean.index.tolist()
    lab_data.df_mean.reset_index(inplace=True)
    lab_data.df_mean['Filter'] = pd.CategoricalIndex(lab_data.df_mean['Filter'], ordered=True, categories=cats)
    lab_data.df_mean.sort_values('Filter', inplace=True)
    lab_data.df_mean.set_index('Filter', inplace=True)

    # client calculation part
    BgdMC1_Before = client_data.df_before_mean['Current1(pA)'].values[0]
    BgdIC1_Before = client_data.df_before_mean['Current2(pA)'].values[0]

    MC1 = (client_data.df_mean['Current1(pA)'] - BgdMC1_Before).to_frame('NK')
    IC1 = (client_data.df_mean['Current2(pA)'] - BgdIC1_Before).to_frame('NK')
    R1 = (client_data.df_mean['Current2(pA)'] - BgdIC1_Before).to_frame('NK') / (client_data.df_mean['Current1(pA)'] - BgdMC1_Before).to_frame('NK')
    TM1 = client_data.df_mean['T(MC)'].to_frame('NK')
    TA1 = client_data.df_mean['T(Air)'].to_frame('NK')

    # lab calculation part
    BgdMC2_Before = lab_data.df_before_mean['Current1(pA)'].values[0]
    BgdIC2_Before = lab_data.df_before_mean['Current2(pA)'].values[0]

    MC2 = (lab_data.df_mean['Current1(pA)'] - BgdMC2_Before).to_frame('NK')
    IC2 = (lab_data.df_mean['Current2(pA)'] - BgdIC2_Before).to_frame('NK')
    R2 = (lab_data.df_mean['Current2(pA)'] - BgdIC2_Before).to_frame('NK') / (lab_data.df_mean['Current1(pA)'] - BgdMC2_Before).to_frame('NK')
    TM2 = lab_data.df_mean['T(MC)'].to_frame('NK')
    TS2 = lab_data.df_mean['T(SC)'].to_frame('NK')
    H2 = lab_data.df_mean['H(%)'].to_frame('NK')

    # read constant and KK from constant excel file
    constant = os.path.join(constant_file_config.get_path())
    df_constant = pd.read_excel(constant, sheet_name='constant', engine='openpyxl')
    df_beams = pd.read_excel(constant, sheet_name='Beams', engine='openpyxl')

    # get ma and WE
    Ma = df_constant['ma'].values[0]
    WE = df_constant['WE'].values[0]

    # reform the KK into the same order as client_data.df_mean so that it will be easy to calculate directly
    # (do not need to extract the same index to do calculation. We can just calculate on data frame).

    # get the filter of the first measurement from KK
    df_beams = df_beams[df_beams.Filter.isin(client_data.df_mean.index)]
    cats = client_data.df_mean.index[:-duplicate_num]
    df_beams['Filter'] = pd.CategoricalIndex(df_beams['Filter'], ordered=True, categories=cats)
    df_beams.sort_values('Filter', inplace=True)

    # get the filter which measure two times from KK
    df_beams_duplicate = df_beams[df_beams.Filter.isin(client_data.df_mean.index[:duplicate_num])]
    cats = client_data.df_mean.index[:duplicate_num]
    df_beams_duplicate['Filter'] = pd.CategoricalIndex(df_beams_duplicate['Filter'], ordered=True, categories=cats)
    df_beams_duplicate['Filter'] = [Filter + '*' for Filter in df_beams_duplicate.Filter.tolist()]
    df_beams_duplicate.sort_values('Filter', inplace=True)

    # concate the processed beams
    df_processBeams = pd.concat([df_beams, df_beams_duplicate], axis=0).set_index('Filter')

    # concat together so that KK will have the same order as client_data.df_mean
    df_KK = df_processBeams['Product'].to_frame('NK')

    # Calculating NK
    NK = R2 * WE * df_KK * ((273.15 + TS2) / (273.15 + TM2)) * (0.995766667 + 0.000045 * H2) / (
                Ma * R1 * (273.15 + TA1) / (273.15 + TM1))
    NK = round(NK / 1000000, 2)
    NK.columns = ['NK']

    # leakage
    df_leakage = pd.DataFrame({"Before": [BgdMC1_Before, BgdIC1_Before, BgdMC2_Before, BgdIC2_Before],
                                  "After": [client_data.df_after_mean['Current1(pA)'].values[0],
                                                       client_data.df_after_mean['Current2(pA)'].values[0],
                                                       lab_data.df_after_mean['Current1(pA)'].values[0],
                                                       lab_data.df_after_mean['Current2(pA)'].values[0]]},
                                 index=["Monitor 1", "Chamber (client)", "Monitor 2", "Standard (MEFAC)"])

    # extract the effective energy for plotting the graph
    df_energy = df_processBeams['E_eff'].to_frame()

    # create an object to save result
    result = result_data()

    # saving NK
    result.df_NK = NK

    # saving leakage
    result.df_leakage = round(df_leakage, 2) + 0  # +0 to remove -0.0

    # get the coordinate which the cell need to be highlighted
    result.highlight = [(x, y) for x, y in zip(*np.where(abs(result.df_leakage.values) > 0.2))]

    # saving the graph required data
    result.X = df_energy['E_eff'].to_frame('E_eff')

    # save other constant to result
    result.df_otherConstant = df_processBeams.drop(columns=['Product', 'E_eff']).fillna('')

    return result

def extraction(path):
    """
    This function use to extract the measurement data from the file.

    :param : path: the path of file
    :return: return the measurement data which is extracted from the file. Besides, return the number of
             beams quality which measure two times in order to use in the calculation part
    """
    Backgrounds_num = 0
    Measurements_num = 0
    df_total = None

    with open(path, newline='', encoding="ISO-8859-1") as f:
        for line in f:
            if 'DATA' in line:
                break
            # get the Background number
            elif 'Backgrounds' in line:
                Backgrounds_num = int(line.split(',')[2])

            # get the measurement number
            elif 'Measurements' in line:
                Measurements_num = int(line.split(',')[2])
        df_total = pd.read_csv(f)

    df_total.drop(['kV', 'mA', 'HVLFilter(mm)', 'N', 'P(kPa)', 'Comment'], axis=1, inplace=True)

    # change the column type to numeric
    df_total = df_total.apply(pd.to_numeric, errors='ignore')

    # dataframe of the before xray data
    df_before = df_total[:Backgrounds_num]

    # dataframe of the beam data with open xray
    df_data = df_total[Backgrounds_num:-Backgrounds_num]

    # check the number of measurement for each beam
    df_size = df_data.groupby(['Filter']).size().to_frame('count')

    # seperate the beam which measure two times
    duplicate_beam = list(df_size[df_size['count'] > Measurements_num].index)

    df_noDuplicate = df_data[~df_data.Filter.isin(duplicate_beam)]
    df_duplicate = df_data[df_data.Filter.isin(duplicate_beam)]
    df_first = df_duplicate[:Measurements_num * len(duplicate_beam)]
    df_last = df_duplicate[-Measurements_num * len(duplicate_beam):]

    # put * on the last measurements (which measure two times) of beams' name
    df_last.Filter = [Filter + '*' for Filter in df_last.Filter.tolist()]

    # dataframe of the after xray data
    df_after = df_total[-Backgrounds_num:]

    # average the data and store to object
    data = Processed_data()

    # put the filter which measure two times at the last
    data.df_mean = pd.concat([df_first.groupby(['Filter']).mean(), df_noDuplicate.groupby(['Filter']).mean(),
                              df_last.groupby(['Filter']).mean()], axis=0)

    data.df_before_mean = df_before.groupby(['Filter']).mean()
    data.df_after_mean = df_after.groupby(['Filter']).mean()

    # return number of duplicate_beam since kk need to know how many beams measure two times
    return data, len(duplicate_beam)

def summary(name_list, result_list):
    """
    This function is used to summary all runs of calibration coefficient.
    This function can deal with if the runs do not have same beam quality

    :param name_list  : a list of all names of runs. ex: ['run1', 'run2']
    :param result_list: a list of all NK of runs
    :param df_energy  : a dataframe of the effective energy
    :return           : return a dataframe of the summary
    """
    # get the first dataframe in order to merge with others
    df_result = result_list[0].df_NK

    # merge all results' dataframes into one dataframe
    for i in range(1, len(result_list)):
        df_result = pd.merge(df_result, result_list[i].df_NK, left_index=True, right_index=True, how='outer')

    # give the column name and average the all rows
    df_result.columns = name_list
    df_result['Average'] = round(df_result.mean(axis=1), 2)

    # all results' values divided by the average value
    for name in name_list:
        df_result[name + '/Average'] = round(df_result[name] / df_result['Average'], 3).map('{:,.3f}'.format)
        df_result[name] = df_result[name].map('{:,.2f}'.format)

    # let the format be consisted in the GUI
    df_result['Average'] = df_result['Average'].map('{:,.2f}'.format)

    # deal with effective energy
    df_energy = result_list[0].X

    # merge the effective energy with summary
    df_summary = pd.merge(df_energy, df_result, left_index=True, right_index=True, how='outer')

    # sort by voltage
    df_summary['Tube voltage'] = [re.findall(r'\d+', beam)[0] for beam in df_summary.index]
    df_summary['Tube voltage'] = pd.to_numeric(df_summary['Tube voltage'])

    # find how many beams have measure two times
    duplicate_num = len([beam for beam in df_summary.index if '*' in beam])

    df_first = df_summary[:-duplicate_num].sort_values(by='Tube voltage')
    df_second = df_summary[-duplicate_num:].sort_values(by='Tube voltage')

    df_first_sort = None
    for voltage in df_first['Tube voltage'].unique():
        df_temp = df_first[df_first['Tube voltage'] == voltage]  # extract one voltage
        df_temp.sort_values(by='E_eff', inplace=True)  # sort by energy
        df_first_sort = pd.concat([df_first_sort, df_temp], axis=0)  # concate with other voltage

    df_first_sort['E_eff'] = df_first_sort['E_eff'].apply('{:,.2f}'.format)
    df_second['E_eff'] = df_second['E_eff'].apply('{:,.2f}'.format)

    df_summary = pd.concat([df_first_sort, df_second], axis=0)

    return df_summary.drop(columns='Tube voltage')

def pdf_visualization(path, df_summary, df_otherConstant):
    """
    Merge all the constants and NK together. Besides, the data will be placed by voltage and order by effective energy.
    Finally, it will produce the tables and the pictures based on the dataframe

    :param df_summary: Dataframe - the summary of the selected runs
    :param df_otherConstant: Dataframe - the other constants of the beam quality. Since all run will have same beams,
                             only one df_otherConstant needs to be passed
    :return: Dataframe - will return a dataframe which consist all the constants and NK
    """
    # using dataframe.merge rather than pd.merge is because the index order will be preserved
    df_merge = df_summary['E_eff'].to_frame('Nominal effective energy [1]').reset_index()
    df_merge = df_merge.merge(df_otherConstant.reset_index(), how='outer')  # merge energy with other constants

    # merge energy + other constants with NK
    df_merge = df_merge.merge(df_summary['Average'].to_frame('NK [2]').reset_index(), how='outer').set_index("Filter")

    # get the voltage for every beam
    df_merge['Tube voltage'] = [re.findall(r'\d+', beam)[0] for beam in df_merge.index]
    df_merge['Tube voltage'] = pd.to_numeric(df_merge['Tube voltage'])

    # Order the column as needs
    df_merge.index.name = 'Beam code'
    df_merge = df_merge.iloc[:, [7, 1, 2, 3, 4, 0, 5, 6]]

    # hard code U % value
    df_merge['U %'] = 1.4
    df_merge = df_merge.apply(pd.to_numeric).fillna('')
    ################################################### Draw kVp ####################################################
    color = cm.rainbow(np.linspace(0, 1, len(df_merge['Tube voltage'].unique())))
    plt.rcParams["figure.figsize"] = (8, 4)

    plot1 = plt.figure(1)
    for voltage, c in zip(df_merge['Tube voltage'].unique(), color):
        df_temp = df_merge[df_merge['Tube voltage'] == voltage]  # extract one voltage
        plt.plot(df_temp['Tube voltage'], df_temp['NK [2]'], marker='D', label=str(voltage) + ' kVp', c=c)

    plt.xlabel("kVp", fontweight='bold')
    plt.ylabel(r'$\bfN_K$ (mGy/nc)', fontweight='bold')
    plt.grid(axis='y', linestyle='--')
    plt.legend(ncol=2)
    plt.gca().yaxis.set_major_formatter(StrMethodFormatter('{x:,.1f}'))

    plot1.savefig(path + r'\kVp.png', bbox_inches='tight', dpi=1200)
    ################################################### Draw kVp ####################################################
    ################################################## Draw HVL Al ##################################################
    plot2 = plt.figure(2)
    for voltage, c in zip(df_merge['Tube voltage'].unique(), color):
        df_temp = df_merge[df_merge['Tube voltage'] == voltage]  # extract one voltage
        df_temp = df_temp[~(df_temp['HVL(mm Al)'] == '')]  # remove the HVL value which is ''
        if len(df_temp) == 0:
            continue
        plt.plot(np.round(df_temp['HVL(mm Al)'].to_list(), 2), df_temp['NK [2]'], '.-', label=str(voltage) + ' kVp', c=c)

    plt.xlabel(r'HVL (mm Al)', fontweight='bold')
    plt.ylabel(r'$\bfN_K$ (mGy/nc)', fontweight='bold')
    plt.grid(axis='y', linestyle='--')
    plt.gca().xaxis.set_major_formatter(StrMethodFormatter('{x:,.2f}'))
    plt.gca().yaxis.set_major_formatter(StrMethodFormatter('{x:,.1f}'))
    plt.legend(ncol=2)
    plt.rcParams["figure.figsize"] = (8, 4)
    plot2.savefig(path + r'\HVL_Al.png', bbox_inches='tight', dpi=1200)
    ################################################## Draw HVL Al ##################################################
    ################################################## Draw HVL Cu ##################################################
    plot3 = plt.figure(3)
    for voltage, c in zip(df_merge['Tube voltage'].unique(), color):
        df_temp = df_merge[df_merge['Tube voltage'] == voltage]  # extract one voltage
        df_temp = df_temp[~(df_temp['HVL(mm Cu)'] == '')]  # remove the HVL value which is ''
        if len(df_temp) == 0:
            continue
        plt.plot(np.round(df_temp['HVL(mm Cu)'].to_list(), 2), df_temp['NK [2]'], '.-', label=str(voltage) + ' kVp', c=c)

    plt.xlabel(r'HVL (mm Cu)', fontweight='bold')
    plt.ylabel(r'$\bfN_K$ (mGy/nc)', fontweight='bold')
    plt.grid(axis='y', linestyle='--')
    plt.gca().xaxis.set_major_formatter(StrMethodFormatter('{x:,.2f}'))
    plt.gca().yaxis.set_major_formatter(StrMethodFormatter('{x:,.1f}'))
    plt.legend(ncol=2)
    plt.rcParams["figure.figsize"] = (8, 4)
    plot3.savefig(path + r'\HVL_Cu.png', bbox_inches='tight', dpi=1200)

    plt.close('all')
    ################################################## Draw HVL Cu ##################################################
    ################################################## Save to excel ###################################################
    first_table_num = 40
    subset_beams = ['NXA50', 'NXA70', 'NXB100', 'NXC120', 'NXD140', 'NXE150', 'NXF200', 'NXG250', 'NXH280', 'NXH300',
                    'NXH300*']

    df_merge = df_merge.apply(pd.to_numeric, errors='ignore')  # change '' to nan so that na_action can work
    df_merge['Added filter(mm Al)'] = df_merge['Added filter(mm Al)'].map('{:,.1f}'.format, na_action='ignore')
    df_merge['Added filter(mm Cu)'] = df_merge['Added filter(mm Cu)'].map('{:,.1f}'.format, na_action='ignore')
    df_merge['HVL(mm Al)'] = df_merge['HVL(mm Al)'].map('{:,.2f}'.format, na_action='ignore')
    df_merge['HVL(mm Cu)'] = df_merge['HVL(mm Cu)'].map('{:,.2f}'.format, na_action='ignore')
    df_merge['Nominal air kerma rate'] = df_merge['Nominal air kerma rate'].map('{:,.1f}'.format, na_action='ignore')

    df_subset = df_merge[df_merge.index.isin(subset_beams)].style.set_properties(**{'text-align': 'center'})
    df_merge1 = df_merge.iloc[:first_table_num].style.set_properties(**{'text-align': 'center'})
    df_merge2 = df_merge.iloc[first_table_num:].style.set_properties(**{'text-align': 'center'})

    table_path = path + r'\pdf_table.xlsx'
    writer = pd.ExcelWriter(table_path, engine='xlsxwriter')
    df_subset.to_excel(writer, sheet_name='subset')
    df_merge1.to_excel(writer, sheet_name='total-1')
    df_merge2.to_excel(writer, sheet_name='total-2')
    writer.save()
    writer.close()
    ################################################## Save to excel ###################################################

def extractionHeader(client_path: str, lab_path: str):
    """
    This function is used to extract the run info data from the file.
    """
    # check run num in file name (Disabled)

    # client_file_name = client_path.split('//')[-1]
    # lab_file_name = lab_path.split('//')[-1]
    # searchObj = re.search( r'Run[0-9]*', client_file_name, re.I)
    # run = searchObj.group()
    # if not re.search( run, lab_file_name, re.I):
    #     raise HeaderError("Not the same run")

    client_data = Header_data()
    lab_data = Header_data()

    # read data from header
    with open(client_path, newline='', encoding="ISO-8859-1") as f:
        for line in f:
            if 'DATA' in line:
                break
            elif 'CAL Number' in line:
                client_data.CAL_num = line.split(',')[2].strip()
            elif 'Client name' in line:
                client_data.Client_name = line.split(',')[2].strip()
            elif 'Address 1' in line:
                client_data.address_1 = line.split(',')[2].strip()
            elif 'Address 2' in line:
                client_data.address_2 = line.split(',')[2].strip()
            elif 'Operator' in line:
                client_data.operator = line.split(',')[2].strip()
            elif 'Chamber' in line:
                chamber = line.split(',')[2].strip()
                if not re.match(r'(.+) (.+)', chamber):
                    raise HeaderError("Client file does not contains model/serial, please check!")
                else:
                    client_data.serial = chamber.split()[-1]
                    client_data.model = " ".join(chamber.split()[:-1])
            elif 'Date' in line:
                client_data.date = line.split(',')[2].strip()
                # client_data.date = datetime.strptime(dateStr, "%m/%d/%Y").date()

    with open(lab_path, newline='', encoding="ISO-8859-1") as f:
        for line in f:
            if 'DATA' in line:
                break
            elif 'CAL Number' in line:
                lab_data.CAL_num = line.split(',')[2].strip()
            elif 'Client name' in line:
                lab_data.Client_name = line.split(',')[2].strip()
            elif 'Address 1' in line:
                lab_data.address_1 = line.split(',')[2].strip()
            elif 'Address 2' in line:
                lab_data.address_2 = line.split(',')[2].strip()
            elif 'Operator' in line:
                lab_data.operator = line.split(',')[2].strip()
            elif 'Chamber' in line:
                lab_data.model = line.split(',')[2].strip()

    # data integrity check
    if client_data.model + client_data.serial == "MEFAC":
        raise HeaderError("Client file is a MEFAC run, please check!")
    elif lab_data.model != "MEFAC":
        raise HeaderError("Lab file is not MEFAC run, please check!")
    elif client_data.CAL_num == "":
        raise HeaderError("Client file does not contains CAL num, please check!")
    elif lab_data.CAL_num == "":
        raise HeaderError("Lab file does not contains CAL num, please check!")
    elif client_data.serial == "" or client_data.model == "":
        raise HeaderError("Client file does not contains model/serial, please check!")
    # elif not re.match(r"\d{4}", client_data.serial):
    #     raise HeaderError("Invalid chamber in client file, please check!")
    elif lab_data.CAL_num != client_data.CAL_num:
        raise HeaderError("CAL num from Client file and Lab file not the same, please check!")

    return client_data


# encoding=utf-8
import os
import web
import xtemplate
import xutils
import xconfig

C_TEMPLATE = """
#include <stdio.h>
#include <string.h>
#include <stdlib.h>

int main(int argc, char* argv) {
    printf("hello,world!");
    return 0;
}
"""

class TccHandler:
    def GET(self):
        args = web.input(code=None, json=None)
        code = args.code
        json = args.json
        output = ""
        if code is None:
            code = C_TEMPLATE
        else:
            path = os.path.join(xconfig.TMP_DIR, "temp.c")
            xutils.savetofile(path, code)
            status, output = xutils.getstatusoutput("D:\\tcc\\tcc.exe -run %s" % path)

            if json == "true":
                return xutils.json_str(status=status, output=output)
        return xtemplate.render("tools/tcc.html",
            show_aside = False,
            code = code,
            output = output)

    def POST(self):
        return self.GET()

class handler:

    def GET(self, name):
        name = xutils.unquote(name)
        if not name.endswith(".html"):
            name += ".html"
        # Chrome下面 tools/timeline不能正常渲染
        web.header("Content-Type", "text/html")
        fpath = os.path.join(xconfig.HANDLERS_DIR, "tools", name)
        if os.path.exists(fpath):
            return xtemplate.render("tools/" + name, show_aside = False)
        else:
            raise web.notfound()

    def POST(self, name):
        return self.GET(name)

xurls = (r"/tools/tcc", TccHandler)


# coding=utf-8
from models import Lesson, Homeworkcommit, Course, Studentcourse, Coursehomework
from school.models import Student
from constant import *
from django.http import HttpResponse
from django.core.urlresolvers import reverse
from django.shortcuts import redirect, render
import json
from course.auth import has_course_permission
from django.db.models import F
from django.conf import settings
from wechat.client import wechat_client
from course.tasks import send_homework_remind

def startLesson(request):
    lessondata = Lesson.objects.get(id=request.GET.get('lessonid'))
    if not has_course_permission(request.user, lessondata.course):
        if request.is_ajax():
            return HttpResponse(json.dumps({'error': 101, 'message': '没有权限'}), content_type="application/json")
        else:
            return render(request, 'error.html',
                          {'message': '没有权限'})
    if request.is_ajax():
        return HttpResponse(json.dumps(lessondata.startlesson()), content_type="application/json")
    else:
        re = lessondata.startlesson()
        if re['error'] != 0:
            return render(request, 'error.html',
                          {'message': re['message'],
                           'submessage': lessondata.course.title,
                           'jumpurl': str(
                               reverse('course:information', args=[lessondata.course.id]))})
        else:
            if request.GET.get('jumptocheckin', default=0) == '1':
                return redirect(reverse('checkin:startcheckin', args=[lessondata.id]))
            else:
                return redirect(reverse('checkin:lesson_data', args=[lessondata.id]))

def stopLesson(request):
    lessondata = Lesson.objects.get(id=request.GET.get('lessonid'))
    if not has_course_permission(request.user, lessondata.course):
        if request.is_ajax():
            return HttpResponse(json.dumps({'error': 101, 'message': '没有权限'}), content_type="application/json")
        else:
            return render(request, 'error.html',
                          {'message': '没有权限'})
    if request.is_ajax():
        return HttpResponse(json.dumps(lessondata.stoplesson()), content_type="application/json")
    else:
        re = lessondata.stoplesson()
        if re['error'] != 0:
            return render(request, 'error.html',
                          {'message': re['message'],
                           'submessage': lessondata.course.title,
                           'jumpurl': str(
                               reverse('course:information', args=[lessondata.course.id]))})
        else:
            return redirect(reverse('course:information', args=[lessondata.course.id]))

def clearLesson(request):
    lessondata = Lesson.objects.get(id=request.GET.get('lessonid'))
    if not has_course_permission(request.user, lessondata.course):
        if request.is_ajax():
            return HttpResponse(json.dumps({'error': 101, 'message': '没有权限'}), content_type="application/json")
        else:
            return render(request, 'error.html',
                          {'message': '没有权限'})
    if request.is_ajax():
        return HttpResponse(json.dumps(lessondata.cleardata()), content_type="application/json")
    else:
        re = lessondata.cleardata()
        if re['error'] != 0:
            return render(request, 'error.html',
                          {'message': re['message'],
                           'submessage': lessondata.course.title,
                           'jumpurl': str(
                               reverse('course:information', args=[lessondata.course.id]))})
        else:
            return redirect(reverse('course:information', args=[lessondata.course.id]))

def sethomeworkscore(request):
    homeworkcommit = Homeworkcommit.objects.select_related('coursehomework').get(id=request.GET.get('homeworkcommitid'))
    score = request.GET.get('score')
    if not score.isdigit():
        return HttpResponse(json.dumps({'error': 101, 'message': '分数只能设置为数字'}), content_type="application/json")
    if not has_course_permission(request.user, homeworkcommit.coursehomework.course):
        return HttpResponse(json.dumps({'error': 101, 'message': '没有权限'}), content_type="application/json")
    homeworkcommit.score = score
    homeworkcommit.save()
    return HttpResponse(json.dumps({'error': 0, 'score': score, 'message': '评分成功'}), content_type="application/json")

def leavehomeworkcomment(request):
    homeworkcommit = Homeworkcommit.objects.select_related('coursehomework').get(
        id=request.POST.get('homeworkcommitid'))
    comment = request.POST.get('comment', '')
    if comment == '':
        return HttpResponse(json.dumps({'error': 101, 'message': '留言不能为空'}), content_type="application/json")
    if not has_course_permission(request.user, homeworkcommit.coursehomework.course):
        return HttpResponse(json.dumps({'error': 101, 'message': '没有权限'}), content_type="application/json")
    homeworkcommit.comment = comment
    homeworkcommit.save()
    # send wechat notification
    article = {
        "title": u"作业留言提醒!",
        "description": u"教师留言：" + comment,
        "url": "%s%s" % (settings.DOMAIN, reverse('course:homework', args=[
            homeworkcommit.coursehomework.course_id]) + '?homeworkid=%d' % homeworkcommit.coursehomework_id),
        "image": "%s/static/img/homework.png"
    }
    wechat_client.message.send_articles(agent_id=settings.AGENTID, user_ids=[homeworkcommit.student.user.openid],
                                        articles=[article])
    return HttpResponse(json.dumps({'error': 0, 'message': '留言成功'}), content_type="application/json")

def setperformance_score(request, courseid):
    course = Course.objects.get(id=courseid)
    student = Student.objects.get(studentid=request.GET.get('studentid'))
    score = request.GET.get('score')
    if not has_course_permission(request.user, course):
        return HttpResponse(json.dumps({'error': 101, 'message': '没有权限'}), content_type="application/json")
    sc = Studentcourse.objects.get(student=student, course=course)
    sc.performance_score += int(score)
    sc.save()
    return HttpResponse(json.dumps({'error': 0, 'message': '加分成功', 'performance_score': sc.performance_score}),
                        content_type="application/json")

def remind_homework(request, homeworkid):
    homework = Coursehomework.objects.select_related('course').get(id=homeworkid)
    if not has_course_permission(request.user, homework.course):
        return HttpResponse(json.dumps({'error': 101, 'message': '没有权限'}), content_type="application/json")
    send_homework_remind.delay(homework.id)
    return HttpResponse(json.dumps({'error': 0, 'message': '催交成功'}), content_type="application/json")


import logging
from . import logger, verify_apps, run, vbox

def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('apps', nargs='+')
    args = parser.parse_args()

    handler = logging.StreamHandler()
    logger.addHandler(handler)
    logger.setLevel(logging.DEBUG)
    logger.addHandler(handler)

    vbox_logger = logging.getLogger('vbox')
    vbox_logger.setLevel(logging.DEBUG)
    vbox_logger.addHandler(handler)
    run(*[app for app in verify_apps(vbox.VBox(extraPath=['/usr/local/bin']), *args.apps)])

if __name__ == "__main__":
    main()


from ..utils import *
from ..trainval import manage_dirs_drybean

from .ax import AugEplanation, compute_co_score, softmax

DEBUG_TOGGLES = {
    'val_iter': 0,
    'vis_iter': 0,
}
import warnings
warnings.filterwarnings('ignore')
# remove warning about captum removing forward, backward hooks

def ax_drybean_test(dargs, ax_method=None):
    print('ax_drybean_test')
    model = 'drybeanFPA'

    DIRS = manage_dirs_drybean(dargs)

    n_class = 7
    from ..other_data import DryBeanDataset
    testdataset = DryBeanDataset(DIRS['REC_TEST_DATA_DIR'], DIRS['DRYBEAN_NORMALIZATION_DIR'])
    from torch.utils.data import DataLoader
    testloader = DataLoader(testdataset, batch_size=1, shuffle=False)

    from ..model import drybeanFPA
    net = drybeanFPA().to(device=device)
    ckpt = torch.load(DIRS['MODEL_DIR'])
    net.load_state_dict(ckpt['model_state_dict'])
    net.eval()

    ax = AugEplanation(method=ax_method, model=model)

    label = f"{dargs['data']}_{model}_{str(dargs['ax_method'])}"

    result = {}
    if os.path.exists(DIRS['DRYBEAN_AX_CLASS_ACC_DIR']):
        with open(DIRS['DRYBEAN_AX_CLASS_ACC_DIR']) as f:
            result = json.load(f)

    if ax_method is not None:
        co_scores = []

    n = len(testloader)
    n_correct = 0
    with torch.no_grad():
        for i, (x,y0) in enumerate(testloader):
            x = x.to(torch.float).to(device=device)
            y0 = y0.to(device=device)

            if ax_method is None:
                y = net(x)
            else:
                y_baseline = net(x)

                attr = ax.get_attribution_by_method(net, x)
                attr = ax.normalize_attr(attr)
                y = net(x + attr)

            y_pred = torch.argmax(y,dim=1)[0].item()

            isCorrect = int(y_pred)==int(y0.item())
            if isCorrect:
                n_correct+=1

            if ax_method is not None:
                co_score = compute_co_score(y_baseline, y, y0, n_class)
                co_score_softmax = compute_co_score(y_baseline, softmax(y), y0, n_class)
                co_score_info = (co_score, co_score_softmax, isCorrect)
                co_scores.append(co_score_info)

            if (i+1)%10==0 or (i+1)==n:
                print('%s/%s acc:%s/%s=%s'%(str(i+1),str(n),
                    str(n_correct),str(i+1),str(n_correct/(i+1))),end='\r')

            if DEBUG_TOGGLES['val_iter']:
                if i+1>=20: break

    result[label] = {'n_correct': n_correct, 'n_total': i+1, 'acc':n_correct/(i+1)}
    with open(DIRS['DRYBEAN_AX_CLASS_ACC_DIR'], 'w') as json_file:
        json.dump(result, json_file, indent=4, sort_keys=True)

    if ax_method is not None:
        joblib.dump({'co_scores':co_scores,'ax_method': ax_method}, DIRS['DRYBEAN_CO_SCORE_DIR'])

def vis_co_score(dargs):
    print('vis_co_score')

    DIRS = manage_dirs_drybean (dargs)

    model = 'drybeanFPA'
    from .ax_vis import save_vis_dir, save_acc_table
    save_vis_dir(DIRS['PROJECT_DIR'], DIRS['VIS_DIR'],
        DEBUG_TOGGLE_VIS_ITER=DEBUG_TOGGLES['vis_iter'])
    save_acc_table( dargs['data'], model,
        DIRS['DRYBEAN_AX_CLASS_ACC_DIR'] , DIRS['VIS_DIR'])


from sqlalchemy import Column, ForeignKey, Integer, String
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
from sqlalchemy import create_engine
from flask_login import UserMixin
from flask_dance.consumer.backend.sqla import OAuthConsumerMixin

Base = declarative_base()

class User(UserMixin, Base):
    __tablename__ = 'user'

    id = Column(Integer, primary_key=True)
    username = Column(String(250), unique=True)

class OAuth(OAuthConsumerMixin, Base):
    __tablename__ = 'oauth'

    user_id = Column(Integer, ForeignKey(User.id))
    user = relationship(User)

class Category(Base):
    __tablename__ = 'category'

    id = Column(Integer, primary_key=True)
    name = Column(String(250), nullable=False)

    @property
    def serialize(self):
        return {
            'name': self.name,
            'id': self.id,
        }

class Item(Base):
    __tablename__ = 'item'

    id = Column(Integer, primary_key=True)
    name = Column(String(80), nullable=False)
    description = Column(String(250))
    category_id = Column(Integer, ForeignKey('category.id'))
    category = relationship(Category)

    @property
    def serialize(self):
        return {
            'name': self.name,
            'description': self.description,
            'id': self.id,
            'category_id': self.category_id
        }

engine = create_engine('sqlite:///catalog.db')
Base.metadata.create_all(engine)


# -*- coding: utf-8 -*-
"""
Module provides game itself. \n
It is enough to execute main to get game working\n
"""

import random as r
import time
from random import shuffle
from typing import List

import numpy
import pygame
from pygame.math import Vector2
from scipy.interpolate import interp1d

from background import Background
from meteor import Meteor
from rocket import Rocket
from text_styler import create_text

colors = {
    'red': (255, 0, 0), 'black': (0, 0, 0),
    'white': (255, 255, 255), 'green': (0, 255, 0),
    'brown': (128, 0, 0), 'pink': (255, 51, 153),
    'yellow': (255, 255, 0), 'blue': (0, 0, 255),
    'purple': (153, 0, 255)
}

class Game:
    """ Main module responsible for initialization and handling game itself """
    def __init__(self, size: tuple, rocket: Rocket, background: Background, sprites: List[List[pygame.Surface]]):
        """
        Initializer of a Game class
        :param size: size of game window
        :type size: tuple
        :param rocket: Rect object representing rocket
        :param sprites: list of images representing rocket look
        """
        self.max_x, self.max_y = size
        self.rocket = rocket
        self.background = background
        self.sprites = sprites
        self.game_option = [1, 0, 0, 0]
        self.font = ["Comic Sans MS"]
        self.gravity = 10
        self.air_res = 0.8
        self.pos = None
        self.acc = None
        self.clock = pygame.time.Clock()
        self.move = 0.5
        self.tick_time = 60
        self.background_color = colors['white']
        self.list_of_meteors = []
        self.wind = 0
        self.maximum_number_of_meteors = 20
        self.main_menu_loop = True
        self.is_playing_game = False
        self.game_started = False
        self.is_force = False
        self.is_lost = False
        self.is_won = False
        self.pause = False
        self.win_time = time.time()
        self.count_down_time = 2

    def init_pos(self):
        """
        Method used to initialize rocket position
        """
        self.pos = Vector2(self.start_position, self.surface[self.start_position] - 69)
        self.rocket.x, self.rocket.y = self.pos

    def reset_acc(self):
        """
        Method responsible for resetting acceleration of a rocket
        """
        self.acc = Vector2(0, 0)

    def main_menu(self, screen: pygame.Surface):
        """
        This creates main menu while loop. It's actually the main windows of the game.
        :param screen: Screen on which main menu should be displayed
        """
        texts = ['Start game', 'Options', 'About', 'Exit']
        text_len = len(texts)
        option = 0
        while self.main_menu_loop:
            self.clock.tick(self.tick_time)
            for event in pygame.event.get():
                if event.type == pygame.KEYDOWN:
                    if event.key == pygame.K_DOWN:
                        option = (option + 1) % text_len
                    elif event.key == pygame.K_UP:
                        option = (option - 1) % text_len
                    elif event.key in (pygame.K_SPACE, pygame.K_RETURN):
                        if option == 0:
                            self.play_game(screen)
                        elif option == 1:
                            self.option_menu(screen)
                        elif option == 2:
                            self.about_menu(screen)
                        elif option == 3:
                            self.main_menu_loop = False
                            self.is_playing_game = True
                    elif event.key == pygame.K_ESCAPE:
                        self.is_playing_game = True
                        self.main_menu_loop = False
                elif event.type == pygame.QUIT:
                    self.is_playing_game = True
                    self.main_menu_loop = False

            screen.fill(self.background_color)
            for i, text in enumerate(texts):
                if i == option:
                    color = (0, 155, 0)
                else:
                    color = (0, 0, 255)
                font_size = 60
                new_text = create_text(text, self.font, font_size, color)
                x_pos = (self.max_x - new_text.get_width()) // 2
                y_pos = (1 + i) * (self.max_y - new_text.get_height()) // (2 + text_len) + font_size
                screen.blit(new_text, (x_pos, y_pos))
            pygame.display.flip()

    def about_menu(self, screen: pygame.Surface):
        """
        Menu option showing authors of this marvelous game
        :param screen: about menu will be shown on specified screen
        """
        is_about = True
        authors = ['Kornel Raczak', 'Pawel Gorecki', 'Lukasz Polakiewicz']
        shuffle(authors)
        texts = ['Authors:'] + authors + ['Press any key to escape...']
        text_len = len(texts)
        while is_about:
            self.clock.tick(self.tick_time)
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    is_about = False
                    self.main_menu_loop = False
                elif event.type == pygame.KEYDOWN:
                    if event.key in (pygame.K_SPACE, pygame.K_RETURN, pygame.K_ESCAPE):
                        is_about = False

            screen.fill(self.background_color)
            about_colors = [color for color in colors.values() if color != self.background_color]
            for i, text in enumerate(zip(texts, about_colors)):
                font_size = 60
                if text[0] == texts[-1]:
                    font_size = 30
                new_text = create_text(text[0], self.font, font_size, text[1])
                x_pos = (self.max_x - new_text.get_width()) // 2
                y_pos = (1 + i) * (self.max_y - new_text.get_height()) // (2 + text_len) + font_size
                screen.blit(new_text, (x_pos, y_pos))
            pygame.display.flip()

    def change_wind(self):
        """ Method responsible for changing the force of wind"""
        if self.game_option[3] and not self.game_option[2]:
            self.wind += (r.random() - 0.5) / 700

    def set_wind_on_start(self):
        """ Initializes wind force at the beginning of a game"""
        if not self.game_option[2]:
            self.wind = (r.random() * 2 - 1) / 100

    def set_maximum_number_of_meteors(self):
        """ Sets max number of meteors shown on display """
        self.maximum_number_of_meteors = 12 + 4 * self.game_option[0]

    def option_menu(self, screen: pygame.Surface):
        """
        Creates option menu, where user can configure game play
        :param screen: Screen on which main menu should be displayed
        """
        not_chosen = True
        option_meteors = ["Meteors: Easy", "Meteors: Medium", "Meteors: Hard", "Meteors: Impossible"]
        options1 = ['Music: Off', 'Music: On']
        options2 = ['Wind: On', 'Wind: Off']
        options3 = ['Wind: Constant', 'Wind: Unstable']
        options = [option_meteors, options1, options2, options3]
        game_option_len = []
        for option in options:
            game_option_len.append(len(option))
        option = 0
        text_len = len(options)
        while not_chosen:
            self.clock.tick(self.tick_time)
            for event in pygame.event.get():
                if event.type == pygame.KEYDOWN:
                    if event.key == pygame.K_DOWN:
                        option = (option + 1) % text_len
                    if event.key == pygame.K_UP:
                        option = (option - 1) % text_len
                    elif event.key == pygame.K_LEFT:
                        self.game_option[option] = (self.game_option[option] - 1) % game_option_len[option]
                    elif event.key == pygame.K_RIGHT:
                        self.game_option[option] = (self.game_option[option] + 1) % game_option_len[option]

                    elif event.key == pygame.K_ESCAPE:
                        not_chosen = False
                elif event.type == pygame.QUIT:
                    not_chosen = False
                    self.main_menu_loop = False

            screen.fill(self.background_color)
            for i, opt in enumerate(options):
                color = (0, 0, 255)
                font_size = 60
                what_to_write = opt[self.game_option[i]]
                if i == option:
                    color = (0, 255, 0)
                new_text = create_text(what_to_write, self.font, font_size, color)
                x_pos = (self.max_x - new_text.get_width()) // 2
                y_pos = (1 + i) * (self.max_y - new_text.get_height()) // (2 + text_len) + font_size
                screen.blit(new_text, (x_pos, y_pos))
            pygame.display.flip()

    def show_speed(self, screen: pygame.Surface):
        """
        Method responsible for showing vertical and horizontal speed of rocket, but also the speed of wind.
        :param screen: Screen on which above speeds will be shown
        """
        lagrangian = 117 / 17
        speed_x_val = abs(round(self.acc.x * lagrangian))
        speed_y_val = abs(round(self.acc.y * lagrangian))
        wind = abs(round(self.wind * lagrangian))
        if self.wind > 0:
            wind_direction = ' <-'
        else:
            wind_direction = ' ->'
        speed_x = create_text('Horizontal:  {} m/h'.format(speed_x_val), self.font, 20, (255, 0, 0))
        speed_y = create_text('Vertical:  {} m/h'.format(speed_y_val), self.font, 20, (255, 0, 0))
        speed_wind = create_text('Wind:' + wind_direction + ' {} m/h'.format(10 * wind), self.font, 20, (255, 0, 0))
        screen.blit(speed_x, (10, 30))
        screen.blit(speed_y, (10, 10))
        screen.blit(speed_wind, (10, 50))

    def generate_surface(self, rng: int):
        """
        Generates planet surface along with landing platforms.
        :param rng: Sets maximum height of planets' surface
        """
        self.surface = [self.max_y - 10] * self.max_x
        self.landing_site = []
        self.prepend = 100
        self.site_size = 150
        for i in range(500):
            main_terrain = numpy.random.randint(700, 900)
            self.surface += self.generate_main_terrain(main_terrain, rng, self.surface[self.prepend - 1])
            self.surface += [self.surface[-1]] * self.site_size
            self.landing_site += [
                [self.prepend + self.site_size + main_terrain, self.prepend + 2 * self.site_size + main_terrain]]
        self.start_position = self.prepend + self.site_size // 2 - self.rocket.width // 8
        self.surface = self.surface[(self.max_x - self.prepend - self.site_size):]
        self.cutscene()

    def fill_surface(self, screen: pygame.Surface):
        """
        Fills surface with colour.
        :param screen: Screen on which surface is drawn
        """
        for x in range(0, self.max_x, 3):
            pygame.draw.line(screen, (x // 10 % 256, x // 40 % 256, (x // 20 + 20) % 256), [x, self.max_y],
                             [x, self.surface[x]], 5)

    def cutscene(self):
        """ Moves surface to the right to enable landing on yet another platform. """
        self.surface = self.surface[(self.landing_site[0][0] - self.prepend):]
        self.landing_site = self.landing_site[1:]

    def generate_main_terrain(self, main_terrain: int, rng: int, start: int) -> List[int]:
        """
        Generates curve of terrain in between two platforms.
        :param main_terrain: length of terrain to be generated
        :param rng: maximum height of terrain to be generated
        :param start: heigt of platform preceeding generated piece of terrain
        """
        x = numpy.linspace(0, main_terrain, 20)
        y = [self.max_y - numpy.random.randint(10, rng) for i in range(20)]
        f = interp1d(x, y, kind='cubic')
        return list(f(range(main_terrain)))

    def run(self):
        """ Main method of a Game class. It's responsible for initialization of pygame module and main_menu method. """
        pygame.init()
        pygame.display.set_caption('Mathematicians into space')
        path = r'icon.png'
        icon = pygame.image.load(path)
        icon.set_colorkey((0, 255, 0))
        pygame.display.set_icon(icon)
        screen = pygame.display.set_mode((self.max_x, self.max_y))
        self.background.image = self.background.image.convert()
        self.main_menu(screen)

    def reset_meteors(self):
        """ Clears space from meteors and adds some meteors on start"""
        self.list_of_meteors = []
        how_much_on_start = [6, 9, 12, 15]
        for i in range(how_much_on_start[self.game_option[0]]):
            self.add_meteor()

    def reset_wind(self):
        """ Resets wind """
        self.wind = 0

    def pause_game(self, screen: pygame.Surface):
        """
        Pauses game, so that player can go for a drink
        :param screen: Screen on which main menu should be displayed
        """
        game_pause = create_text("Game Pause", self.font, 90, colors['red'])
        pause_menu = True
        while pause_menu:
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    exit()
                if pygame.key.get_pressed()[pygame.K_ESCAPE]:
                    pause_menu = False
            self.show_text(screen, game_pause, 1 / 2, 1 / 4)
            pygame.display.flip()
            self.clock.tick(30)

    def play_game(self, screen: pygame.Surface):
        """
        Turns on window with game play itself. This method is responsible for all the fun you have!
        :param screen: Screen on which main menu should be displayed
        """
        self.is_playing_game = True
        self.generate_surface(0.5 * self.max_y)
        self.reset_stage()
        self.set_maximum_number_of_meteors()

        counting_texts = [create_text(text, self.font, 165, colors['red']) for text in ['3', '2', '1', 'Go!']]
        self.win_time = time.time()
        while self.is_playing_game:
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    exit()
                if pygame.key.get_pressed()[pygame.K_ESCAPE]:
                    self.pause_game(screen)
            pressed = pygame.key.get_pressed()
            current_time = time.time()
            if self.prob_of_new_meteor():
                self.add_meteor()
            self.overload_meteors()
            self.change_wind()
            self.change_acc(pressed, current_time)
            if self.game_started:
                self.add_gravity()
                self.add_wind()
            self.add_air_resistance()
            self.change_rocket_position()
            screen.fill(self.background_color)
            screen.blit(self.background.image, self.background.rect)
            pygame.draw.lines(screen, colors['white'], 0, [(i, self.surface[i]) for i in range(self.max_x)], 10)
            self.fill_surface(screen)
            self.draw_meteors(screen)
            self.show_speed(screen)
            self.draw_rocket(screen, current_time)
            self.set_game_status()
            if self.is_won:
                self.game_won(screen)
            elif self.is_lost:
                self.game_lost(screen)
            self.show_counting_down(counting_texts, screen, current_time)
            self.clock.tick(self.tick_time)
            pygame.display.flip()

    def show_counting_down(self, counting_texts: List[pygame.Surface], screen: pygame.Surface, current_time: float):
        """
        Method displays counting down at the beginning of every challenge!
        :param counting_texts: texts that should be displayed
        :param screen: Screen on which counting should be displayed
        :param current_time: current time
        """
        time_diff = current_time - self.win_time
        if time_diff <= self.count_down_time:
            text_num = int(1.8 * time_diff)
            self.show_text(screen, counting_texts[text_num], 1 / 2, 1 / 2)

    def game_lost(self, screen: pygame.Surface):
        """
        Method is responsible for handling lost game
        :param screen: main screen
        """
        if self.game_option[1]:
            s = pygame.mixer.Sound('death.wav')
            s.play()
        self.game_over(screen)

    def prob_of_new_meteor(self):
        """ Generates new meteors with probability depending on option set in option menu """
        probabilities = [0.01, 0.03, 0.06, 0.1]
        return r.random() < probabilities[self.game_option[0]]

    def draw_meteors(self, screen: pygame.Surface):
        """
        Method responsible for displaying meteors on the screen and for move of meteors
        :param screen: main screen
        """
        for i, meteor in enumerate(self.list_of_meteors):
            self.list_of_meteors[i].x += meteor.x_acc - self.wind
            self.list_of_meteors[i].y += meteor.y_acc
            self.list_of_meteors[i].rotation_counter = (meteor.rotation_counter + 1) % meteor.rotation_speed
            if self.list_of_meteors[i].rotation_counter == 0:
                self.list_of_meteors[i].rotation = (meteor.rotation + 1) % 36
            scale_size = {70: -1, 60: -2, 50: -3, 40: -4, 30: -5}
            image_meteor = self.sprites[scale_size[meteor.size]][meteor.rotation]
            self.meteor = pygame.Rect(meteor.x, meteor.y, image_meteor.get_width(), image_meteor.get_height())
            screen.blit(image_meteor, self.meteor)

    def overload_meteors(self):
        """
        If meteor is out of the sight it deletes it
        """
        for meteor in self.list_of_meteors:
            if not meteor.is_visible(self.max_x, self.max_y):
                self.list_of_meteors.remove(meteor)

    def add_meteor(self):
        """
        Add new meteor to the list if it is possible
        """
        if len(self.list_of_meteors) < self.maximum_number_of_meteors:
            brzeg = 100
            if r.random() < 0.7:
                y = -brzeg
                x = r.random() * self.max_x
                x_acc = r.random() * 3 - 1.5
                y_acc = r.random() * 2 + 1
            else:
                if r.random() < 0.5:
                    y = r.random() * self.max_y / 3
                    x = self.max_x + brzeg
                    x_acc = - r.random() * 3 - 1
                    y_acc = r.random() * 3 - 1
                else:
                    y = r.random() * self.max_y / 4
                    x = -brzeg
                    x_acc = r.random() * 3 + 1
                    y_acc = r.random() * 3 - 2.5
            rotation = 0
            rotation_speed = r.randint(1, 3)
            rotation_counter = 0
            size = r.choice([70, 60, 50, 40, 30])
            rotation_direction = r.choice([-1, 1])
            meteor = Meteor(x, y, x_acc, y_acc, rotation, rotation_counter, rotation_speed, rotation_direction, size)
            self.list_of_meteors.append(meteor)

    def set_game_status(self):
        """
        Method responsible for checking if game should end (because of win or lose) and setting appropriate flags
        """
        if not 0 < self.rocket.x < self.max_x - self.rocket.width:
            self.is_lost = True
        elif not 0 < self.rocket.y < self.max_y - self.rocket.height:
            self.is_lost = True
        for x, y in self.rocket.hit_box:
            if not y <= self.surface[x]:
                if abs(self.acc.x + self.acc.y) < 5 and \
                        self.landing_site[0][0] < self.rocket.x < self.landing_site[0][1]:
                    self.is_won = True
                    break
                self.is_lost = True
                break
        self.is_meteor_collided()

    def is_meteor_collided(self):
        """
        Check that is collision between rocket and any meteor
        """
        for meteor in self.list_of_meteors:
            for x, y in self.rocket.hit_box:
                if ((meteor.x + meteor.size) - x) ** 2 + ((meteor.y + meteor.size) - y) ** 2 <= 0.81 * meteor.size ** 2:
                    self.is_lost = True

    def change_rocket_position(self):
        """ Method enables changing position of a rocket if not pause and game started """
        if not self.pause and self.game_started:
            self.pos += self.acc

    def add_air_resistance(self):
        """ Adds air resistance to the rocket acceleration """
        if not self.pause:
            self.acc *= (1 - self.air_res / self.tick_time)

    def add_gravity(self):
        """ Adds gravity force to pull down the rocket """
        if not self.pause:
            self.acc.y += self.gravity / self.tick_time

    def add_wind(self):
        """ Adds wind force """
        self.acc.x -= self.wind

    def change_acc(self, pressed: List, current_time: float):
        """
        Method responsible for changing acceleration of a rocket depending on pressed keys.
        :param pressed: pygame list with pressed keys
        :param current_time: current time flag preserves move before game starts
        """
        self.pause = False
        self.is_force = False
        if pressed[pygame.K_UP] and current_time - self.win_time > self.count_down_time:
            self.acc.y -= self.move
            self.is_force = True
            if not self.game_started:
                self.game_started = True
                self.acc.y -= 3
        if pressed[pygame.K_DOWN]:
            if self.acc.y < 0:
                self.acc.y = min(0, self.acc.y + self.move // 4)
        if pressed[pygame.K_LEFT]:
            if self.acc.y != 0:
                self.acc.x -= self.move / 2
        if pressed[pygame.K_RIGHT]:
            if self.acc.y != 0:
                self.acc.x += self.move / 2
        if pressed[pygame.K_SPACE]:
            self.pause = True

    def show_text(self, screen: pygame.Surface, text: pygame.Surface, x_ratio: float, y_ratio: float):
        """
        Method responsible for showing text on screen.
        :param screen: main display
        :param text: text we want to show
        :param x_ratio: x / max_x, ratio where should middle of a text appear according to x axis
        :param y_ratio: y / max_y, ratio where should middle of a text appear according to y axis
        """
        x_pos = x_ratio * (self.max_x - text.get_width())
        y_pos = y_ratio * (self.max_y - text.get_height())
        screen.blit(text, (x_pos, y_pos))

    def game_over(self, screen: pygame.Surface):
        """
        Menu which will be shown after losing the game.
        :param screen: main display
        """
        game_over_text = create_text("Game Over", self.font, 110, (255, 0, 0))
        continue_text = create_text("Press space to restart game", self.font, 35, (255, 0, 0))
        back_to_menu = create_text("Press escape to back to menu", self.font, 35, (255, 0, 0))

        self.show_text(screen, game_over_text, 1 / 2, 1 / 4)
        self.show_text(screen, continue_text, 1 / 2, 4 / 11)
        self.show_text(screen, back_to_menu, 1 / 2, 5 / 11)
        while self.is_lost:
            pygame.display.flip()
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    exit()
                elif event.type == pygame.KEYDOWN:
                    if event.key == pygame.K_SPACE:
                        self.reset_stage()
                    elif event.key == pygame.K_ESCAPE:
                        self.is_lost = False
                        self.is_playing_game = False
                        self.game_started = False
            self.clock.tick(self.tick_time)

    def reset_stage(self):
        """ Resets settings of current game """
        self.init_pos()
        self.reset_acc()
        self.reset_meteors()
        self.game_started = False
        self.is_lost = False
        self.is_won = False
        self.reset_wind()
        self.set_wind_on_start()
        self.win_time = time.time()

    def game_won(self, screen: pygame.Surface):
        """
        Planned behaviour after landing on landing site
        :param screen: main display
        """
        if self.game_option[1]:
            s = pygame.mixer.Sound('victory.wav')
            s.play()
        self.cutscene()
        self.reset_stage()

    def draw_rocket(self, screen: pygame.Surface, current_time: float):
        """
        Method responsible for drawing rocket in the right position
        :param screen: main display
        :param current_time: preserves image change before game start
        """
        image = self.rocket.get_image(self.acc, self.is_force, (current_time - self.win_time) > 3)
        self.rocket.set_pos(self.pos.x, self.pos.y)
        self.rocket.set_width(image.get_width())
        self.rocket.set_height(image.get_height())
        screen.blit(image, self.rocket)


from typing import Callable, List, TypeVar

StringFunc = Callable[[], str]
T = TypeVar('T')
Constructor = Callable[[], T]

def get(of: Constructor[T], *,
        stdin: StringFunc = input,
        strip: bool = True) -> T:
    """Get value of type 'of' from 'stdin'."""

    raw_input = stdin().strip() if strip else stdin()
    return of(raw_input)

def get_list(of: Constructor[T], *,
             stdin: StringFunc = input,
             strip: bool = True,
             delimiter: str = ' ') -> List[T]:
    """Get list of values of type 'of' from 'stdin'."""

    raw_input = stdin().strip() if strip else stdin()
    delimited_raw_input = raw_input.split(delimiter)
    return list(map(of, delimited_raw_input))


import numpy as np
import matplotlib.pyplot as plt
from copy import deepcopy

class MotionPrimitive():
    """
    A motion primitive that defines a trajectory from a over a time T.
    Put functions that all MPs should have in this base class.
    If the implementation is specific to the subclass, raise a NotImplementedError
    """

    def __init__(self, start_state, end_state, num_dims, max_state, subclass_specific_data={'dynamics': None}):
        """
        """
        self.start_state = np.array(start_state).astype(float)
        self.end_state = np.array(end_state).astype(float)
        self.num_dims = np.array(num_dims).astype(int)
        self.max_state = np.array(max_state).astype(float)
        # TODO probably cleaner to do this with kwargs
        self.subclass_specific_data = deepcopy(subclass_specific_data)
        self.n = self.start_state.shape[0]
        self.control_space_q = int(self.n/self.num_dims)
        self.is_valid = False
        self.cost = None
        self.traj_time = None
        # TODO error out if start/end state violate max_state

    @classmethod
    def from_dict(cls, dict, num_dims, max_state, subclass_specific_data={}):
        """
        load a motion primitive from a dictionary
        """
        if dict:
            mp = cls.__new__(cls)
            MotionPrimitive.__init__(mp, np.array(dict["start_state"]),
                                     np.array(dict["end_state"]),
                                     num_dims, max_state, subclass_specific_data)
            mp.cost = dict["cost"]
            mp.traj_time = dict["traj_time"]
            mp.is_valid = True
        else:
            mp = None
        return mp

    def to_dict(self):
        """
        Write important attributes of motion primitive to a dictionary
        """
        if self.is_valid:
            dict = {"cost": self.cost,
                    "traj_time": self.traj_time,
                    "start_state": self.start_state.tolist(),
                    "end_state": self.end_state.tolist(),
                    }
        else:
            dict = {}
        return dict

    def get_state(self, t):
        """
        Given a time t, return the state of the motion primitive at that time.
        Will be specific to the subclass, so we raise an error if the subclass has not implemented it
        """
        raise NotImplementedError

    def get_sampled_states(self, step_size=0.1):
        """
        Return an array consisting of sample times and a sampling of the trajectory for plotting
        Will be specific to the subclass, so we raise an error if the subclass has not implemented it
        """
        raise NotImplementedError

    def get_sampled_position(self, step_size=0.1):
        """
        Return a sampling of only position of trajectory
        Will be specific to the subclass, so we raise an error if the subclass has not implemented it
        """
        raise NotImplementedError

    def get_input(self, t):
        """
        Given a time t, return the input of the motion primitive at that time.
        Will be specific to the subclass, so we raise an error if the subclass has not implemented it
        """
        raise NotImplementedError

    def get_sampled_input(self, step_size=0.1):
        """
        Return a sampling of inputs needed to move along the mp
        Will be specific to the subclass, so we raise an error if the subclass has not implemented it
        """
        raise NotImplementedError

    def plot_from_sampled_states(self, sampling_array, position_only=False, ax=None, color=None, zorder=1):
        """
        Plot time vs. position, velocity, acceleration, and jerk
        """
        # Plot the state over time.
        if color is None:
            color = 'lightgrey'
        if not position_only:
            fig, axes = plt.subplots(4, 1, sharex=True)
            axes[3].set_xlabel('time')
            fig.suptitle('Full State over Time')
            axes[0].set_ylabel("Position")
            axes[1].set_ylabel("Velocity")
            axes[2].set_ylabel("Acceleration")
            axes[3].set_ylabel("Jerk")
            labels = ['x', 'y', 'z']
            for j in range(self.control_space_q+1):
                for i in range(self.num_dims):
                    axes[j].plot(sampling_array[0, :], sampling_array[j*self.num_dims+i+1, :], label=labels[i])
                    if (j > 0):
                        axes[j].plot([0, sampling_array[0, -1]], [self.max_state[j], self.max_state[j]], 'k--')
                        axes[j].plot([0, sampling_array[0, -1]], [-self.max_state[j], -self.max_state[j]], 'k--')
            axes[0].legend()

        else:
            if ax is None:
                ax = plt.gca()
            samples = sampling_array[1:1+self.num_dims, :]
            if self.num_dims == 2:
                ax.plot(samples[0, :], samples[1, :], color=color, zorder=zorder)
            elif self.num_dims == 3:
                ax.plot(samples[0, :], samples[1, :], samples[2, :], color=color, zorder=zorder)

    def plot(self, position_only=False, ax=None, color=None, zorder=1, step_size=.1):
        """
        Generate the sampled state and input trajectories and plot them
        """
        state_sampling = self.get_sampled_states(step_size=step_size)
        _, input_sampling = self.get_sampled_input(step_size=step_size)
        if state_sampling is not None:
            sampling_array = state_sampling
            if input_sampling is not None:
                sampling_array = np.vstack((state_sampling, input_sampling))
            self.plot_from_sampled_states(sampling_array, position_only, ax, color, zorder)
        else:
            print("Trajectory was not found")

    def __eq__(self, other):
        if other is None:
            if self.__dict__.keys() == None:
                return True
            else:
                return False
        if self.__dict__.keys() != other.__dict__.keys():
            return False
        return all(np.array_equal(self.__dict__[key], other.__dict__[key]) for key in self.__dict__ if key != 'subclass_specific_data')

    def __lt__(self, other):
        return self.cost < other.cost

    def __le__(self, other):
        return self.cost <= other.cost

    def __gt__(self, other):
        return self.cost > other.cost

    def __ge__(self, other):
        return self.cost >= other.cost


import logging
import os
import datetime
import socket
import copy

from stellar_base.keypair import Keypair

from ..design import Node

log = logging.getLogger(__name__)

def subparser(subparser):
    parser = subparser.add_parser(
        'fix-design',
        help='fix the incorrect design',
    )
    parser.set_defaults(command='fix_design')

    return

def run(parser, args):
    nodes = copy.deepcopy(args.design.design_yaml['nodes'])
    for k, v in nodes.items():
        if v is None:
            v = Node.get_defaults_design()
        elif 'secret_seed' not in v:
            v['secret_seed'] = Keypair.random().seed().decode()

        args.design.design_yaml['nodes'][k] = v

    print('''%(sep)s
# generated at %(date)s from '%(user)s@%(hostname)s'
%(content)s''' % dict(
        sep='#' * 80,
        date=datetime.datetime.now().isoformat(),
        user=os.environ.get('USER'),
        hostname=socket.gethostname(),
        content=args.design.serialize(),
    ))

    return 0


# Exercise: Convolution and ReLU

# Introduction
# In this exercise, you'll work on building some intuition around feature extraction.
# irst, we'll walk through the example we did in the tutorial again, but this time, with a kernel you choose yourself.
# We've mostly been working with images in this course, but what's behind all of the operations we're learning about
# is mathematics. So, we'll also take a look at how these feature maps can be represented instead as arrays of numbers
# and what effect convolution with a kernel will have on them.

# Run the cell below to get started!
# Setup feedback system
from learntools.core import binder
binder.bind(globals())
from learntools.computer_vision.ex2 import *

import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt

plt.rc('figure', autolayout=True)
plt.rc('axes', labelweight='bold', labelsize='large',
       titleweight='bold', titlesize=18, titlepad=10)
plt.rc('image', cmap='magma')

tf.config.run_functions_eagerly(True)

# Apply Transformations
# The next few exercises walk through feature extraction just like the example in the tutorial.
# Run the following cell to load an image we'll use for the next few exercises.
image_path = '../input/computer-vision-resources/car_illus.jpg'
image = tf.io.read_file(image_path)
image = tf.io.decode_jpeg(image, channels=1)
image = tf.image.resize(image, size=[400, 400])

img = tf.squeeze(image).numpy()
plt.figure(figsize=(6, 6))
plt.imshow(img, cmap='gray')
plt.axis('off')
plt.show();
# You can run this cell to see some standard kernels used in image processing.
import learntools.computer_vision.visiontools as visiontools
from learntools.computer_vision.visiontools import edge, bottom_sobel, emboss, sharpen

kernels = [edge, bottom_sobel, emboss, sharpen]
names = ["Edge Detect", "Bottom Sobel", "Emboss", "Sharpen"]

plt.figure(figsize=(12, 12))
for i, (kernel, name) in enumerate(zip(kernels, names)):
    plt.subplot(1, 4, i+1)
    visiontools.show_kernel(kernel)
    plt.title(name)
plt.tight_layout()

# 1) Define Kernel
# Use the next code cell to define a kernel. You have your choice of what kind of kernel to apply.
# One thing to keep in mind is that the sum of the numbers in the kernel determines how bright the final image is.
# Generally, you should try to keep the sum of the numbers between 0 and 1
# (though that's not required for a correct answer).

# In general, a kernel can have any number of rows and columns. For this exercise, let's use a  3×3
# kernel, which often gives the best results. Define a kernel with tf.constant.
# YOUR CODE HERE: Define a kernel with 3 rows and 3 columns.
kernel = tf.constant([
    [-1, -1, -1],
    [-1,  8, -1],
    [-1, -1, -1],
])
# Uncomment to view kernel
visiontools.show_kernel(kernel)

# Check your answer
q_1.check()

# Now we'll do the first step of feature extraction, the filtering step.
# First run this cell to do some reformatting for TensorFlow.
# Reformat for batch compatibility.
image = tf.image.convert_image_dtype(image, dtype=tf.float32)
image = tf.expand_dims(image, axis=0)
kernel = tf.reshape(kernel, [*kernel.shape, 1, 1])
kernel = tf.cast(kernel, dtype=tf.float32)

# 2) Apply Convolution
# Now we'll apply the kernel to the image by a convolution. The layer in Keras that does this is layers.Conv2D.
# What is the backend function in TensorFlow that performs the same operation?
# YOUR CODE HERE: Give the TensorFlow convolution function (without arguments)
conv_fn = tf.nn.conv2d

# Check your answer
q_2.check()

# Once you've got the correct answer, run this next cell to execute the convolution and see the result!
image_filter = conv_fn(
    input=image,
    filters=kernel,
    strides=1, # or (1, 1)
    padding='SAME',
)

plt.imshow(
    # Reformat for plotting
    tf.squeeze(image_filter)
)
plt.axis('off')
plt.show();
# Can you see how the kernel you chose relates to the feature map it produced?

# 3) Apply ReLU
# Now detect the feature with the ReLU function. In Keras, you'll usually use this as the activation function
# in a Conv2D layer. What is the backend function in TensorFlow that does the same thing?
# YOUR CODE HERE: Give the TensorFlow ReLU function (without arguments)
relu_fn = tf.nn.relu

# Check your answer
q_3.check()

# Once you've got the solution, run this cell to detect the feature with ReLU and see the result!

# The image you see below is the feature map produced by the kernel you chose.
# If you like, experiment with some of the other suggested kernels above, or, try to invent one that will extract
# a certain kind of feature.
image_detect = relu_fn(image_filter)

plt.imshow(
    # Reformat for plotting
    tf.squeeze(image_detect)
)
plt.axis('off')
plt.show();

# In the tutorial, our discussion of kernels and feature maps was mainly visual.
# We saw the effect of Conv2D and ReLU by observing how they transformed some example images.

# But the operations in a convolutional network (like in all neural networks) are usually defined through mathematical
# functions, through a computation on numbers. In the next exercise, we'll take a moment to explore this point of view.

# Let's start by defining a simple array to act as an image, and another array to act as the kernel.
# Run the following cell to see these arrays.
# Sympy is a python library for symbolic mathematics. It has a nice
# pretty printer for matrices, which is all we'll use it for.
import sympy
sympy.init_printing()
from IPython.display import display

image = np.array([
    [0, 1, 0, 0, 0, 0],
    [0, 1, 0, 0, 0, 0],
    [0, 1, 0, 0, 0, 0],
    [0, 1, 0, 0, 0, 0],
    [0, 1, 0, 1, 1, 1],
    [0, 1, 0, 0, 0, 0],
])

kernel = np.array([
    [1, -1],
    [1, -1],
])

display(sympy.Matrix(image))
display(sympy.Matrix(kernel))
# Reformat for Tensorflow
image = tf.cast(image, dtype=tf.float32)
image = tf.reshape(image, [1, *image.shape, 1])
kernel = tf.reshape(kernel, [*kernel.shape, 1, 1])
kernel = tf.cast(kernel, dtype=tf.float32)

# 4) Observe Convolution on a Numerical Matrix
# What do you see? The image is simply a long vertical line on the left and a short horizontal line on the lower right.
# What about the kernel? What effect do you think it will have on this image? After you've thought about it,
# run the next cell for the answer.

# In the tutorial, we talked about how the pattern of positive numbers will tell you the kind of features the kernel
# will extract. This kernel has a vertical column of 1's, and so we would expect it to return features of vertical
# lines.

# Now let's try it out. Run the next cell to apply convolution and ReLU to the image and display the result.
image_filter = tf.nn.conv2d(
    input=image,
    filters=kernel,
    strides=1,
    padding='VALID',
)
image_detect = tf.nn.relu(image_filter)

# The first matrix is the image after convolution, and the second is
# the image after ReLU.
display(sympy.Matrix(tf.squeeze(image_filter).numpy()))
display(sympy.Matrix(tf.squeeze(image_detect).numpy()))
# Is the result what you expected?

# Conclusion
# In this lesson, you learned about the first two operations a convolutional classifier uses for feature extraction:
# filtering an image with a convolution and detecting the feature with the rectified linear unit.


import sqlite3
from som_logg import (LoginCreate, Prof, helper)
conn = sqlite3.connect('sample.db')
cur = conn.cursor()
cur.execute('''CREATE TABLE IF NOT EXISTS users(login TEXT, pasw TEXT, alternative TEXT)''')

conn.commit()

cmd = input('Введите команду, которую хотите использовать: \n')

command_dict = {'/help': helper,
                '/new_user': LoginCreate,
                '/selector': Prof}

def income_func():
    if cmd == '/help':
        command_dict['/help']()
    if cmd == '/new_user':
        command_dict['/new_user']('sample.db').create_user()
    if cmd == '/selector':
        command_dict['/selector'](cur, 'users').get_info_user()

income_func()


# mod parser 3d-news for TorrentPier-II 2.1.5 alpha
# ver 1.0 by dr_brown

from urllib.request import Request, urlopen
from urllib.error import HTTPError
from urllib.parse import quote
from bs4 import BeautifulSoup
from time import sleep
import pymysql.cursors
import urllib
import time
import re

'''If use re-uploading images to your image hosting, set True (only chevereto support)'''
image_api = False
key_api = ''  # api key
url_api = ''  # url image hosting

network_news_forum_id = 2  # id forum news
bot_uid = -746
user_agent = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ' \
             'Chrome/78.0.3904.70 Safari/537.36'

global_url = 'https://3dnews.ru/'
url = 'https://3dnews.ru/news'

'''Connect to Database'''
db_connect = dict(host=' ', user=' ', password=' ', db=' ', charset='utf8mb4',
                  cursorclass=pymysql.cursors.DictCursor)

connection = pymysql.connect(**db_connect)

def run():
    try:
        with connection.cursor() as cursor:
            page = parse_html(url)
            if page is None:
                return False
            links = parse_links(page, cursor)
            if links is None:
                return False
            for link in links:
                html = parse_html(link)
                if html is None:
                    return False
                data = parse_inner_page(html)
                save_row(data, cursor)
                sleep(1)
    finally:
        connection.close()

def parse_html(parse_url):
    req = Request(parse_url, headers={'User-Agent': user_agent})
    try:
        page = BeautifulSoup(urlopen(req).read().decode('UTF-8'), 'html.parser')
    except HTTPError:
        return None
    return page

def parse_links(page, cursor):
    name_id = []
    try:
        for wrap in page.find_all('div', {'class': 'article-entry article-infeed marker_sw nImp0 nIcat10 cat_10 nIaft '
                                                   'newsAllFeedHideItem'}):
            name_id.append(wrap.get('id'))
    except AttributeError:
        return None
    links = []
    for ids in name_id:
        sql = "SELECT * FROM bb_news_grab WHERE import_id = %s"
        cursor.execute(sql, (ids,))
        result = cursor.fetchone()
        if result is not None:
            continue
        import_sql = "INSERT INTO bb_news_grab (import_id) VALUES (%s)"
        cursor.execute(import_sql, (ids,))
        connection.commit()
        links.append(global_url + ids)
    return links

def parse_inner_page(html):
    try:
        name_text = html.find('div', {'class': 'js-mediator-article'})
    except AttributeError:
        name_text = ''
    else:
        name_text = clear_url(name_text)
    try:
        name_title = html.find('h1', {'itemprop': 'headline'}).get_text('h1')
    except AttributeError:
        name_title = ''
    result = dict(text=name_text, title=name_title)
    return result

def clear_url(name_text):
    if image_api:
        images = name_text.find_all('img')
        upload_image = []
        pic_image = []
        for image in images:
            try:
                image_clr = urllib.parse.quote(image['src'], safe='-._~:/?#[]@!$&\'()*+,;=')
                pic_req = Request(url_api.format(key_api, image_clr))
                pic = urlopen(pic_req).read().decode('UTF-8')
                if pic == 'Invalid file source':
                    continue
                elif pic == 'File too big - max 2 MB':
                    continue
            except HTTPError:
                continue
            else:
                upload_image.append(image['src'])
                pic_image.append(pic)
    regex = r'<a.*?>([\s\S]*?)<\/a>'
    name_text = str(name_text)
    clear_text = re.sub(regex, r'\1', name_text)
    clear_text = re.sub(r'<br\s*?>', r' ', clear_text)
    clear_text = re.sub(r'\s+', r' ', clear_text)
    if image_api:
        for (a, b) in zip(upload_image, pic_image):
            clear_text = re.sub(a, b, clear_text)
    return clear_text

def save_row(data, cursor):
    topic_sql = "INSERT INTO bb_topics (topic_title, topic_poster, topic_time, forum_id, topic_status, " \
                "topic_type, topic_dl_type, topic_vote, topic_last_post_time) VALUES (%s, %s, %s, %s, " \
                "%s, %s, %s, %s, %s)"
    cursor.execute(topic_sql, (data['title'], bot_uid, time.time(), network_news_forum_id, '0', '0',
                               '0', '0', time.time(),))
    connection.commit()
    topic_last_id = "SELECT LAST_INSERT_ID() as last"
    cursor.execute(topic_last_id)
    topic_id = cursor.fetchone()
    post_sql = "INSERT INTO bb_posts (topic_id, forum_id, poster_id, post_username, post_time, " \
               "poster_ip) VALUES (%s, %s, %s, %s, %s, %s)"
    cursor.execute(post_sql, (topic_id['last'], network_news_forum_id, bot_uid, 'Bot', time.time(),
                              '127.0.0.1',))
    connection.commit()
    post_last_id = "SELECT LAST_INSERT_ID() as last"
    cursor.execute(post_last_id)
    post_id = cursor.fetchone()
    update_topic = "UPDATE bb_topics SET topic_first_post_id = %s, topic_last_post_id = %s WHERE " \
                   "topic_id = %s"
    cursor.execute(update_topic, (post_id['last'], post_id['last'], topic_id['last'],))
    post_text_sql = "INSERT INTO bb_posts_text (post_id, post_text) VALUES (%s, %s)"
    cursor.execute(post_text_sql, (post_id['last'], data['text'],))
    update_forum = "UPDATE bb_forums SET forum_posts = forum_posts + 1, forum_last_post_id = %s, " \
                   "forum_topics = forum_topics + 1 WHERE forum_id = %s"
    cursor.execute(update_forum, (post_id['last'], network_news_forum_id,))
    connection.commit()

run()


import numpy as np
from sklearn.metrics import jaccard_similarity_score
import os
from os import listdir
from os.path import isfile, join
import cv2

def load_image(img_org,INPUT_SIZE):
    h, w, c = img_org.shape
    img = cv2.resize(img_org, (INPUT_SIZE, INPUT_SIZE))
    img = np.array(img, dtype=np.float32)
    x = np.expand_dims(img, axis=0)
    x = preprocess_input(x)
    return x

def load_label(img_org,INPUT_SIZE,nb_classes):
    img = cv2.resize(img_org,(INPUT_SIZE, INPUT_SIZE))
    img = np.array(img, dtype=np.uint8)
    img[img==255] = 0
    y = np.zeros((1, img.shape[0], img.shape[1], nb_classes), dtype=np.float32)
    for i in range(img.shape[0]):
        for j in range(img.shape[1]):
            y[0, i, j, img[i][j]] = 1
    return y

def get_IOU(prediction_dir, gt_dir):
    input_size = 321
    nb_classes = 21
    all_imgs= [f for f in listdir(prediction_dir) if isfile(join(prediction_dir, f))]
    predictions, gt = [], []
    for im_name in all_imgs:
        pred_img_org = cv2.imread(os.path.join(prediction_dir, im_name))
        #print(pred_img_org)
        gt_img_org = cv2.imread(gt_dir + im_name[:-9] + ".png")
        #gt_img_org = cv2.imread(os.path.join(gt_dir, im_name))
        #print(gt_dir + im_name[:-9] + ".png")
        predictions.append(load_label(pred_img_org, input_size, nb_classes))
        gt.append(load_label(gt_img_org, input_size, nb_classes))

    iou = jaccard_similarity_score(np.array(gt).flatten(), np.array(predictions).flatten())
    print(iou)

if __name__ == "__main__":
    get_IOU("output/prediction/", "/om2/user/cfmata/voc12/train/VOCdevkit/VOC2012/SegmentationClass_1D")
    #get_IOU("output/deeplab_resnet_ckpt_prediction_test/", "output/deeplab_resnet_ckpt_prediction_test/") #debug


import os
import hashlib
import argparse

# Grab command line arguments for directory and signature file
parser = argparse.ArgumentParser()
parser.add_argument('--directory', '-d', help="The directory to scan for viruses in", required=True)
parser.add_argument('--signatures', '-s', help="The signature file", required=True)
args = parser.parse_args()

print("[*] Scanning directory: %s"%args.directory)
print("[*] Using signature file: %s"%args.signatures)

blocksize=65536

for root, subFolders, files in os.walk(args.directory):
    # Take MD5 Of the file

    for fi in files:
        with open(os.path.join(root, fi), 'rb') as fin:
            hash_md5 = hashlib.md5()
            for block in iter(lambda: fin.read(blocksize), b""):
                hash_md5.update(block)
            digest = hash_md5.hexdigest()
            with open(args.signatures) as signatures_file:
                for sig in signatures_file:
                    if sig.strip() == str(digest):
                        print("[!] File %s is a virus"%(os.path.join(root, fi)))


#####################################################
# Introspective Variational Autoencoder Config File #
#####################################################
# A Brock, 2016
import numpy as np
import matplotlib.pyplot as plt

from keras.utils.visualize_util import plot
from keras.models import Model
from keras import backend as K
from keras import objectives
from keras.datasets import mnist

from keras.models import load_model

from keras.layers import Input, Dense, Convolution2D, MaxPooling2D, UpSampling2D,Lambda
from keras.layers.core import Dense, Dropout, Flatten, Reshape

K.set_image_dim_ordering('tf')

class VAE:

    # Start with a low learning rate then increase learning rate on second epoch
    # lr_schedule = {0: 0.0001, 1: 0.005}
    # Configuration Dictionary
    cfg = {'batch_size': 100,
           # 'learning_rate': lr_schedule,
           'reg': 0.001,
           'momentum': 0.9,
           'input_dim': (28, 28),
           'n_channels': 1,
           'n_classes': 10,
           'max_epochs': 50,
           'latent_dim': 2,
           'intermediate_dim': 32,
           }
    data = []
    enc_dec =[]
    enc =[]
    dec =[]
    rmnist = []

    if K.image_dim_ordering() == 'tf':
        original_img_size = (28, 28, 1)
    else:
        original_img_size = (1, 28, 28)

    def __init__(self):
        self.enc_dec = self.get_model()

    def get_data(self,db='mnist'):
        if db=='mnist':
            (x_train, y_train), (x_test, y_test) = mnist.load_data()
            (x_train, y_train), (x_test, y_test) = mnist.load_data()
            x_train = x_train.astype('float32') / 255.
            x_train = x_train.reshape((x_train.shape[0],) + self.original_img_size)
            x_test = x_test.astype('float32') / 255.
            x_test = x_test.reshape((x_test.shape[0],) + self.original_img_size)
            x_train[x_train < 0.5] = 0
            x_train[x_train >= 0.5] = 1
            x_test[x_test < 0.5] = 0
            x_test[x_test >= 0.5] = 1

            self.data = {
                'x_train': x_train,
                'y_train': y_train,
                'x_test': x_test,
                'y_test': y_test
            }
        else:
            raise AssertionError('db missing now')
        print('loading training data done')

    def get_model(self,interp=False):

        if interp:
            batch_size = self.cfg['batch_size']
            original_dim = np.prod(self.original_img_size)
            latent_dim = self.cfg['latent_dim']
            intermediate_dim = 256
            epsilon_std = 0.01

            x = Input(batch_shape=(batch_size, original_dim))
            h = Dense(intermediate_dim, activation='sigmoid')(x)
            z_mean = Dense(latent_dim)(h)
            z_log_var = Dense(latent_dim)(h)

            def sampling(args):
                z_mean, z_log_var = args
                epsilon = K.random_normal(shape=(batch_size, latent_dim), mean=0.,
                                          std=epsilon_std)
                return z_mean + K.exp(z_log_var / 2) * epsilon

            # note that "output_shape" isn't necessary with the TensorFlow backend
            z = Lambda(sampling, output_shape=(latent_dim,))([z_mean, z_log_var])

            # we instantiate these layers separately so as to reuse them later
            decoder_h = Dense(intermediate_dim, activation='sigmoid')
            decoder_mean = Dense(original_dim, activation='sigmoid')
            h_decoded = decoder_h(z)
            x_decoded_mean = decoder_mean(h_decoded)

            def vae_loss(x, x_decoded_mean):
                xent_loss = original_dim * objectives.binary_crossentropy(x, x_decoded_mean)
                kl_loss = - 0.5 * K.sum(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
                return xent_loss + kl_loss

            vae = Model(x, x_decoded_mean)
            vae.compile(optimizer='adadelta', loss=vae_loss)
        else:
            batch_size = 100
            original_dim = 28 * 28
            latent_dim = 2
            # nb_epoch = 50

            if K.image_dim_ordering() == 'tf':
                x = Input(shape=(28, 28, 1))
            else:
                x = Input(shape=(1, 28, 28))
            # h = Flatten()(x)
            # x = Input(batch_shape=(batch_size, original_dim))
            # h = Reshape((1,28,28))(x)
            encode_h1 = Convolution2D(16, 3, 3, activation='relu', border_mode='same')
            encode_h2 = MaxPooling2D((2, 2), border_mode='same')
            encode_h3 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')
            encode_h4 = MaxPooling2D((2, 2), border_mode='same')
            encode_h5 = Convolution2D(64, 3, 3, activation='relu', border_mode='same')
            encode_h6 = MaxPooling2D((2, 2), border_mode='same')

            h = encode_h1(x)
            h = encode_h2(h)
            h = encode_h3(h)
            h = encode_h4(h)
            h = encode_h5(h)
            h = encode_h6(h)

            h = Flatten()(h)
            z_mean = Dense(latent_dim)(h)
            z_log_var = Dense(latent_dim)(h)
            encoder = Model(x, z_mean)

            def sampling(args):
                z_mean, z_log_var = args
                epsilon = K.random_normal(shape=(batch_size, latent_dim), mean=0.)
                return z_mean + K.exp(z_log_var / 2) * epsilon

            # note that "output_shape" isn't necessary with the TensorFlow backend
            z = Lambda(sampling, output_shape=(latent_dim,))([z_mean, z_log_var])

            # we instantiate these layers separately so as to reuse them later

            decoder_h1 = Dense(64 * 4 * 4, activation='relu')
            if K.image_dim_ordering() == 'tf':
                decoder_h2 = Reshape((4, 4, 64))
            else:
                decoder_h2 = Reshape((64, 4, 4))
            decoder_h3 = Convolution2D(64, 3, 3, activation='relu', border_mode='same')
            decoder_h4 = UpSampling2D((2, 2))
            decoder_h5 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')
            decoder_h6 = UpSampling2D((2, 2))
            decoder_h7 = Convolution2D(16, 3, 3, activation='relu', border_mode='same')
            decoder_h8 = UpSampling2D((2, 2))
            decoder_h9 = Convolution2D(1, 5, 5, activation='sigmoid', border_mode='valid')

            # g_input = Input(shape=[2])
            H = decoder_h1(z)
            H = decoder_h2(H)
            H = decoder_h3(H)
            H = decoder_h4(H)
            H = decoder_h5(H)
            H = decoder_h6(H)
            H = decoder_h7(H)
            H = decoder_h8(H)
            g_V = decoder_h9(H)

            # def g2b(g_V):
            #     '''grey value to binary'''
            #     return K.round(g_V)
            # g_V = Lambda(g2b, output_shape=g_V._keras_shape[1:4])(g_V)

            def vae_loss(x, g_V):
                xent_loss = original_dim * K.mean(objectives.binary_crossentropy(x, g_V))
                kl_loss = - 0.5 * K.sum(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
                return xent_loss + kl_loss

            vae = Model(x, g_V)
            # keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
            vae.compile(optimizer='rmsprop', loss=vae_loss)

            vae.summary()

            return vae

    def train_vae(self):
        x_train = self.data['x_train']
        x_test = self.data['x_test']
        self.enc_dec.fit(x_train, x_train,
                shuffle=True,
                nb_epoch=self.cfg['max_epochs'],
                batch_size=self.cfg['batch_size'],
                validation_data=(x_test, x_test))

    def encoder(self):
        # load from trained vae model
        x = Input(shape=self.original_img_size)
        h = x
        for i in range(1,9):
            h = self.enc_dec.layers[i](h)

        return Model(x, h)

    def decoder(self):
        # load from trained vae model
        x = Input(shape=(self.cfg['latent_dim'],))
        h = x
        for i in range(11,20):
            h = self.enc_dec.layers[i](h)
        return Model(x, h)

    def input2latent(self):
        # display a 2D plot of the digit classes in the latent space
        encoder = vae.encoder()
        x_test_encoded = encoder.predict(self.data['x_test'], batch_size=self.data['batch_size'])
        plt.figure(figsize=(6, 6))
        plt.scatter(x_test_encoded[:, 0], x_test_encoded[:, 1], c=self.data['y_test'])
        plt.colorbar()
        plt.show()

    def latent2output(self):
        # display a 2D manifold of the digits
        n = 15  # figure with 15x15 digits
        digit_size = 28
        figure = np.zeros((digit_size * n, digit_size * n))
        # we will sample n points within [-15, 15] standard deviations
        grid_x = np.linspace(-0.2, 0.2, n)
        grid_y = np.linspace(-0.2, 0.2, n)

        decoder = vae.decoder()
        for i, yi in enumerate(grid_x):
            for j, xi in enumerate(grid_y):
                z_sample = np.array([[xi, yi]])
                x_decoded = decoder.predict(z_sample)
                digit = x_decoded[0].reshape(digit_size, digit_size)
                figure[i * digit_size: (i + 1) * digit_size,
                j * digit_size: (j + 1) * digit_size] = digit

        plt.figure(figsize=(10, 10))
        plt.imshow(figure)
        plt.show(block=True)

    def reconstruction(self):
        dec = self.decoder()
        enc = self.encoder()
        n = 20
        data = self.data['x_train'][0:n]
        enc_x = enc.predict(data)
        rec_x = dec.predict(enc_x)

        plt.figure(figsize=(20, 4))
        for i in range(n):
            # display original
            ax = plt.subplot(2, n, i+1)
            plt.imshow(data[i].reshape(28, 28), cmap='Greys',interpolation='nearest')
            ax.get_xaxis().set_visible(False)
            ax.get_yaxis().set_visible(False)
            ax = plt.subplot(2, n, i + n+1)

            # display reconstruction
            plt.imshow(rec_x[i].reshape(28, 28), cmap='Greys',interpolation='nearest')
            ax.get_xaxis().set_visible(False)
            ax.get_yaxis().set_visible(False)
        plt.show()
        return rec_x

    def saliency(self):
        '''get saliency info'''
        y_train = self.data['y_train']
        oneidx = np.where(y_train == 1)
        rmnist_train = np.zeros((oneidx[0].shape[0], self.cfg['input_dim'][0], self.cfg['input_dim'][1], 1))
        bmnist_train = np.zeros((oneidx[0].shape[0], self.cfg['input_dim'][0], self.cfg['input_dim'][1], 1))
        for idx,idxx in enumerate(oneidx[0]):
            data = self.data['x_train'] [idxx,:,:,0] # only one channel
            cmax = 1.0
            for ypos in range(28):
                if (data[ypos] == 1).any():
                    d = ypos
                    break
            for ypos in range(d, 28):
                if (data[ypos] == 1).any():
                    xpos = np.where(data[ypos] == 1)
                    tmp = cmax - 10.0/255.0 * (ypos - d)
                    for jdx in xpos[0]:
                        rmnist_train[idx,ypos,jdx,0] = tmp
                        bmnist_train[idx,ypos,jdx,0] = 1

        y_test = self.data['y_test']
        oneidx = np.where(y_test == 1)
        rmnist_test = np.zeros((oneidx[0].shape[0], self.cfg['input_dim'][0], self.cfg['input_dim'][1], 1))
        bmnist_test = np.zeros((oneidx[0].shape[0], self.cfg['input_dim'][0], self.cfg['input_dim'][1], 1))
        for idx,idxx in enumerate(oneidx[0]):
            data = self.data['x_test'] [idxx,:,:,0] # only one channel
            cmax = 1.0
            for ypos in range(28):
                if (data[ypos] == 1).any():
                    d = ypos
                    break
            for ypos in range(d, 28):
                if (data[ypos] == 1).any():
                    xpos = np.where(data[ypos] == 1)
                    tmp = cmax - 10.0/255.0 * (ypos - d)
                    for jdx in xpos[0]:
                        rmnist_test[idx,ypos,jdx,0] = tmp
                        bmnist_test[idx,ypos,jdx,0] = 1
        return{
            'bmnist_train': bmnist_train,
            'bmnist_test': bmnist_test,
            'rmnist_train': rmnist_train,
            'rmnist_test': rmnist_test
        }

    def sal_dec(self):
        data = self.saliency()

        sal = self.decoder()
        sal.compile(optimizer='rmsprop', loss='binary_crossentropy')
        sal.summary()

        encoder = self.encoder()
        enc_x_train = encoder.predict(data['bmnist_train'])
        enc_x_test = encoder.predict(data['bmnist_test'])

        sal.fit(enc_x_train, data['rmnist_train'],
                 shuffle=True,
                 nb_epoch=self.cfg['max_epochs'],
                 batch_size=self.cfg['batch_size'],
                 validation_data=(enc_x_test, data['rmnist_test']))

        n = 5
        binput = data['bmnist_train'][0:n,:,:,:]
        routput = sal.predict(encoder.predict(binput))
        plt.figure(figsize=(20, 4))
        for i in range(n):
            # display original
            ax1 = plt.subplot(2, n, i + 1)
            plt.imshow(binput[i,:,:,0].reshape(28, 28),cmap='Greys',interpolation='nearest')
            ax1.get_xaxis().set_visible(False)
            ax1.get_yaxis().set_visible(False)
            # display sal prediction
            ax2 = plt.subplot(2, n, i + 1 + n)
            plt.imshow(routput[i,:,:,0].reshape(28, 28),cmap='Greys',interpolation='nearest')
            ax2.get_xaxis().set_visible(False)
            ax2.get_yaxis().set_visible(False)
        plt.show(block=True)

        binput = data['bmnist_test'][0:n, :, :, :]
        routput = sal.predict(encoder.predict(binput))
        plt.figure(figsize=(20, 4))
        for i in range(n):
            # display original
            ax1 = plt.subplot(2, n, i + 1)
            plt.imshow(binput[i, :, :, 0].reshape(28, 28), cmap='Greys',interpolation='nearest')
            ax1.get_xaxis().set_visible(False)
            ax1.get_yaxis().set_visible(False)
            # display sal prediction
            ax2 = plt.subplot(2, n, i + 1 + n)
            plt.imshow(routput[i, :, :, 0].reshape(28, 28), cmap='Greys',interpolation='nearest')
            ax2.get_xaxis().set_visible(False)
            ax2.get_yaxis().set_visible(False)
        plt.show(block=True)

if 1:
    vae = VAE()
    vae.get_data()
    vae.train_vae()
    #
    #
    # plot(vae.enc_dec, to_file='model.png')
    # vae.enc_dec.save('enc_dec.h5')  # creates a HDF5 file 'my_model.h5'
    #
    vae.reconstruction()

    vae.sal_dec()

else:
    vae = VAE()
    vae.get_data()

    load_model('my_model.h5')
    vae.enc = vae.encoder()
    vae.dec = vae.decoder()

    vae.latent2output()

    vae.reconstruction()