from codecs import open
from textable import table
import numpy as np
import matplotlib.pylab as plt
from scipy.optimize import curve_fit
from uncertainties import ufloat
from uncertainties import unumpy as unp

MU_NULL = 4 * np.pi * 10**-7
E_NULL = 1.6021766208 * 10**-19
MASS_NULL = 9.10938356 * 10**-31
PLANCK_H = 6.626070040 * 10**-34
MU_B = 9.274009994 * 10**-24
DELTA_87 = 4.53 * 10**-24
DELTA_85 = 2.01 * 10**-24

R_SWEEP = 16.39 * 10**-2
N_SWEEP = 11
R_VERT = 11.735 * 10**-2
N_VERT = 20
R_HOR = 15.79 * 10**-2
N_HOR = 154
T = 50 + 273.13

S = 1/2
L = 0
J = 1/2

DELTA_I = 0.5 * 10**-3
DELTA_NU = 0.5

peak1 = 6
peak2 = 11
DELTA_PEAK1 = 0.5
DELTA_PEAK2 = 0.5

peak1 = ufloat(peak1, DELTA_PEAK1)
peak2 = ufloat(peak2, DELTA_PEAK2)

def _calc_b(inputData, N, R):
    B = MU_NULL * 8 * inputData * N / (np.sqrt(125) * R)
    return B

def _calc_average(inputData):
    sumNumbers = 0
    for line in inputData:
        sumNumbers += line
    return sumNumbers / len(inputData)

def func_linear(x, a, b):
    return a*x + b

def _calc_g(m):
    g = 4*np.pi*MASS_NULL / (m*E_NULL)
    return g

def _calc_square(inputB, inputG, deltaJ, mList):
    squareList = []
    singleList = []
    for m in mList:
        deltaValOne = inputG * MU_B * unp.nominal_values(inputB)
        deltaValTwo = (1 - 2*m) / deltaJ
        squareList.append(deltaValOne**2 * deltaValTwo)
        singleList.append(deltaValOne)
    return np.array(singleList), np.array(squareList)

def _calc_gj():
    gjz = 3.0023*J*(J+1) + 1.0023*(S*(S+1) - L*(L+1))
    gjn = 2*J*(J+1)
    return np.float64(gjz / gjn)

def _calc_i(gj, gf):
    I1 = -(2 - gj/(2*gf)) / 2
    I2 = I1**2 - 3/4*(1 - gj/gf)
    I2 = np.sqrt(I2)
    return I1 + I2

def _calc_ratio():
    peak2Val = (1 + peak1/peak2)**-1
    peak1Val = 1 - peak2Val
    return peak1Val, peak2Val

nuRaw, sweep0Raw, hor0Raw, sweep1Raw, hor1Raw, sweep2Raw, \
    hor2Raw = np.genfromtxt('data/aufg_c.dat', unpack=True)

nuArr = unp.uarray(nuRaw * 10**3, DELTA_NU)
sweep0Raw = unp.uarray(sweep0Raw, DELTA_I)
sweep1Raw = unp.uarray(sweep1Raw, DELTA_I)
sweep2Raw = unp.uarray(sweep2Raw, DELTA_I)
hor0Raw = unp.uarray(hor0Raw, DELTA_I)
hor1Raw = unp.uarray(hor1Raw, DELTA_I)
hor2Raw = unp.uarray(hor2Raw, DELTA_I)
with open('build/tex/c_data_raw_0.tex', 'w') as f:
    f.write(
        table(
            [
                r'$\nu / \si{\kilo\hertz}$',
                r'$I_\text{0,sweep} / \si{\milli\ampere}$',
                r'$I_\text{0,hor} / \si{\milli\ampere}$'
            ], [
                nuArr*1e-3,
                sweep0Raw*1e3,
                hor0Raw*1e3
            ]
            )
        )

with open('build/tex/c_data_raw_87.tex', 'w') as f:
    f.write(
        table(
            [
                r'$\nu / \si{\kilo\hertz}$',
                r'$I_\text{87,sweep} / \si{\milli\ampere}$',
                r'$I_\text{87,hor} / \si{\milli\ampere}$'
            ], [
                nuArr*1e-3,
                sweep1Raw*1e3,
                hor1Raw*1e3,
            ]
            )
        )

with open('build/tex/c_data_raw_85.tex', 'w') as f:
    f.write(
        table(
            [
                r'$\nu / \si{\kilo\hertz}$',
                r'$I_\text{85,sweep} / \si{\milli\ampere}$',
                r'$I_\text{85,hor} / \si{\milli\ampere}$'
            ], [
                nuArr*1e-3,
                sweep2Raw*1e3,
                hor2Raw*1e3
            ]
            )
        )

sweep0B = _calc_b(sweep0Raw, N_SWEEP, R_SWEEP)
sweep1B = _calc_b(sweep1Raw, N_SWEEP, R_SWEEP)
sweep2B = _calc_b(sweep2Raw, N_SWEEP, R_SWEEP)

hor0B = _calc_b(hor0Raw*3, N_HOR, R_HOR)
hor1B = _calc_b(hor1Raw*3, N_HOR, R_HOR)
hor2B = _calc_b(hor2Raw*3, N_HOR, R_HOR)

b0Ges = hor0B + sweep0B
b1Ges = hor1B + sweep1B
b2Ges = hor2B + sweep2B

with open('build/tex/c_data_b_0.tex', 'w') as f:
    f.write(
        table(
            [
                r'$\nu / \si{\kilo\hertz}$',
                r'$B_\text{0,ges} / \si{\micro\tesla}$',
                r'$B_\text{0,sweep} / \si{\micro\tesla}$',
                r'$B_\text{0,hor} / \si{\micro\tesla}$'
            ], [
                nuArr*1e-3,
                b0Ges*1e6,
                hor0B*1e6,
                sweep0B*1e6
            ])
        )

with open('build/tex/c_data_b_87.tex', 'w') as f:
    f.write(
        table(
            [
                r'$\nu / \si{\kilo\hertz}$',
                r'$B_\text{87,ges} / \si{\micro\tesla}$',
                r'$B_\text{87,sweep} / \si{\micro\tesla}$',
                r'$B_\text{87,hor} / \si{\micro\tesla}$'
            ], [
                nuArr*1e-3,
                b1Ges*1e6,
                hor1B*1e6,
                sweep1B*1e6
            ])
        )

with open('build/tex/c_data_b_85.tex', 'w') as f:
    f.write(
        table(
            [
                r'$\nu / \si{\kilo\hertz}$',
                r'$B_\text{85,ges} / \si{\micro\tesla}$',
                r'$B_\text{85,sweep} / \si{\micro\tesla}$',
                r'$B_\text{85,hor} / \si{\micro\tesla}$'
            ], [
                nuArr*1e-3,
                b2Ges*1e6,
                hor2B*1e6,
                sweep2B*1e6
            ])
        )

bVert = _calc_b(ufloat(0.26, DELTA_I), N_VERT, R_VERT)
bHor = _calc_average(b0Ges)

with open('build/tex/c_vert.tex', 'w') as f:
    f.write(r'\SI{{{:L}}}{{\micro\tesla}}'.format(bVert*1e6))

with open('build/tex/c_data_erd.tex', 'w') as f:
    f.write(
        table(
            [
                r'$B_\text{Vert} / \si{\micro\tesla}$',
                r'$B_\text{Hor} / \si{\micro\tesla}$'
            ], [
                bVert*1e6,
                bHor*1e6
            ])
        )

popt1, pcov1 = curve_fit(
    func_linear, unp.nominal_values(nuArr),
    unp.nominal_values(b1Ges)
    )
popt2, pcov2 = curve_fit(
    func_linear, unp.nominal_values(nuArr),
    unp.nominal_values(b2Ges)
    )

with open('build/tex/c_fit_m_87.tex', 'w') as f:
    f.write(r'\SI{%1.4e}{\micro\tesla\per\mega\per\hertz}' % (popt1[0]))
with open('build/tex/c_fit_b_87.tex', 'w') as f:
    f.write(r'\SI{%1.4e}{\micro\tesla\per\mega\per\hertz}' % (popt1[1]))
with open('build/tex/c_fit_m_85.tex', 'w') as f:
    f.write(r'\SI{%1.4e}{\micro\tesla\per\mega\per\hertz}' % (popt2[0]))
with open('build/tex/c_fit_b_85.tex', 'w') as f:
    f.write(r'\SI{%1.4e}{\micro\tesla\per\mega\per\hertz}' % (popt2[1]))

gf1 = _calc_g(popt1[0])
gf2 = _calc_g(popt2[0])
gj = _calc_gj()
i1 = _calc_i(gj, gf1)
i2 = _calc_i(gj, gf2)

with open('build/tex/c_I_87.tex', 'w') as f:
    f.write(r'\SI{{{:.4f}}}{{}}'.format(i1))

with open('build/tex/c_I_85.tex', 'w') as f:
    f.write(r'\SI{{{:.4f}}}{{}}'.format(i2))

with open('build/tex/c_data_gi.tex', 'w') as f:
    f.write(
        table(
            [
                r'$g_\text{f,87}$',
                r'$g_\text{f,85}$',
                r'$g_\text{j}$',
                r'$I_\text{87}$',
                r'$I_\text{85}$'
            ], [
                gf1.round(4),
                gf2.round(4),
                gj.round(4),
                i1.round(4),
                i2.round(4)
            ]
            )
        )

mList = np.array([-2, -1, 0, 1, 2])

square1 = np.array(_calc_square(b1Ges, gf1, DELTA_87, mList))
square2 = np.array(_calc_square(b2Ges, gf2, DELTA_85, mList))

deltaSquare1 = square1[1] / square1[0]
deltaSquare2 = square2[1] / square2[0]

mArr = np.array([-2, -1, 0, 1, 2])
mArray = np.empty(len(deltaSquare1[0]))

for m in range(len(mArr)):
    mArray.fill(m - 2)
    with open('build/tex/c_data_square_87_' + str(m) + '.tex', 'w') as f:
        f.write(
            table([
                    r'M$_\text{F}$',
                    r'B$_\text{Ges., 87} / \si{\micro\tesla}$',
                    r'U$_\text{1., 87} / 10^{-30} \si{\joule}$',
                    r'U$_\text{2., 87} / 10^{-33} \si{\joule}$',
                    r'$\frac{\text{U}_\text{2., 87}}{\text{U}_\text{1., 87}} / 10^{-3}$'
                    ], [
                        mArray,
                        b1Ges * 1e6,
                        np.round(square1[0][m] * 1e30, 4),
                        np.round(square1[1][m] * 1e33, 4),
                        np.round(deltaSquare1[m] * 1e3, 4)
                    ])
                )

for m in range(len(mArr)):
    mArray.fill(m-2)
    with open('build/tex/c_data_square_85_' + str(m) + '.tex', 'w') as f:
        f.write(
            table([
                    r'M$_\text{F}$',
                    r'B$_\text{Ges., 85} / \si{\micro\tesla}$',
                    r'U$_\text{1., 85} / 10^{-30} \si{\joule}$',
                    r'U$_\text{2., 85} / 10^{-33} \si{\joule}$',
                    r'$\frac{\text{U}_\text{2., 85}}{\text{U}_\text{1., 85}} / 10^{-3}$'
                    ], [
                        mArray,
                        b2Ges * 1e6,
                        np.round(square2[0][m] * 1e30, 4),
                        np.round(square2[1][m] * 1e33, 4),
                        np.round(deltaSquare2[m] * 1e3, 4)
                    ])
                )

with open('build/tex/c_data_square_87.tex', 'w') as f:
    m = np.array([[m]*10 for m in mList])
    b = np.array([[b1Ges]*5])
    f.write(
        table([
                r'M$_\text{F}$',
                r'B$_\text{Ges., 87} / \si{\micro\tesla}$',
                r'U$_\text{1., 87} / 10^{-27} \si{\joule}$',
                r'U$_\text{2., 87} / 10^{-33} \si{\joule}$',
                r'$\frac{\text{U}_\text{2., 87}}{\text{U}_\text{1., 87}} / 10^{-3}$'
                ], [
                    m.flatten(),
                    b[0].flatten() * 1e6,
                    np.round(square1[0].flatten() * 1e27, 4),
                    np.round(square1[1].flatten() * 1e33, 4),
                    np.round(deltaSquare1.flatten() * 1e3, 4)
                ])
            )

with open('build/tex/c_data_square_85.tex', 'w') as f:
    m = np.array([[m]*10 for m in mList])
    b = np.array([[b2Ges]*5])
    f.write(
        table([
                r'M$_\text{F}$',
                r'B$_\text{Ges., 85} / \si{\micro\tesla}$',
                r'U$_\text{1., 85} / 10^{-27} \si{\joule}$',
                r'U$_\text{2., 85} / 10^{-33} \si{\joule}$',
                r'$\frac{\text{U}_\text{2., 85}}{\text{U}_\text{1., 85}} / 10^{-3}$'
                ], [
                    m.flatten(),
                    b[0].flatten() * 1e6,
                    np.round(square2[0].flatten() * 1e27, 4),
                    np.round(square2[1].flatten() * 1e33, 4),
                    np.round(deltaSquare2.flatten() * 1e3, 4)
                ])
            )

ratio1, ratio2 = _calc_ratio()

with open('build/tex/c_ratio_87.tex', 'w') as f:
    f.write(r'\num{{{:L}}}'.format(peak1))

with open('build/tex/c_ratio_85.tex', 'w') as f:
    f.write(r'\num{{{:L}}}'.format(peak2))

with open('build/tex/c_data_ratio.tex', 'w') as f:
    f.write(
        table([
                r'Rb$_\text{87} / \si{\percent}$',
                r'Rb$_\text{85} / \si{\percent}$'
                ], [
                    ratio1*100,
                    ratio2*100
                ])
            )

popt1, pcov1 = curve_fit(
    func_linear, unp.nominal_values(nuArr)*1e-6,
    unp.nominal_values(b1Ges)*1e6
    )
popt2, pcov2 = curve_fit(
    func_linear, unp.nominal_values(nuArr)*1e-6,
    unp.nominal_values(b2Ges)*1e6
    )

x = np.linspace(
    unp.nominal_values(nuArr[0])*1e-6,
    unp.nominal_values(nuArr[-1])*1e-6, 10**5
    )
plt.plot(x, func_linear(x, *popt1), label=r'Regressionskurve $a(\nu)$')
plt.errorbar(
    unp.nominal_values(nuArr)*1e-6,
    unp.nominal_values(b1Ges)*1e6,
    yerr=unp.std_devs(b1Ges), fmt='kx',
    label=r'Messwerte'
    )
plt.title('Rubidium 87')
plt.ylabel(r'B / $\mu$T')
plt.xlabel(r'$\nu$ / MHz')
plt.tight_layout()
plt.legend(loc='best')
plt.xlim(
    unp.nominal_values(x[0])*90/100,
    unp.nominal_values(x[-1])*101/100
    )
plt.grid()
plt.savefig('build/plots/c_fit_87.pdf')
plt.clf()

plt.plot(x, func_linear(x, *popt2), label=r'Regressionskurve $b(\nu)$')
plt.errorbar(
    unp.nominal_values(nuArr)*1e-6,
    unp.nominal_values(b2Ges)*1e6,
    yerr=unp.std_devs(b2Ges), fmt='kx',
    label=r'Messwerte'
    )
plt.title('Rubidium 85')
plt.ylabel(r'B / $\mu$T')
plt.xlabel(r'$\nu$ / MHz')
plt.tight_layout()
plt.xlim(
    unp.nominal_values(x[0])*90/100,
    unp.nominal_values(x[-1])*101/100
    )
plt.grid()
plt.legend(loc='best')
plt.savefig('build/plots/c_fit_85.pdf')
plt.clf()


from tradingview_ta import TA_Handler, Interval, Exchange

import time
import firebase_admin
from firebase_admin import credentials, firestore
from datetime import datetime
import calendar

kimlik = credentials.Certificate("ema_class.json")

app = firebase_admin.initialize_app(kimlik)
kesisimmacdsayac = 0
kesisimmovaverage = 0
longgiris = 0
shortgiris = 0
sayici_giris_control = 0
sheetsymbolx = 0
sheetssymboly = 0
macdlast = 0
slowmacd = 0
fastmacd =0
class Macdema():

    def __init__(self, symbol, timeframe, quantity, buyvalue, sellvalue):
        self.symbol = symbol
        self.kesisimmacdsayac = kesisimmacdsayac
        self.longgiris = longgiris
        self.shortgiris = shortgiris
        self.timeframe = timeframe
        self.sayici_giris_control = sayici_giris_control
        self.macdlast = macdlast
        self.slowmacd=slowmacd
        self.fastmacd=fastmacd
        self.quantity = quantity
        self.buyvalue = buyvalue
        self.sellvalue = sellvalue

    def dfall(self, symbol, timeframe):

        handler = TA_Handler(
            symbol=symbol,
            exchange="BINANCE",
            screener="crypto",
            interval=timeframe,
            timeout=None,
            proxies = {'http': '47.242.84.173:3128', 'http': '181.205.20.195:999','http': '192.111.135.17:18302' ,'http': '103.108.228.185:7497'}
        )

        analysis = handler.get_analysis().summary

        analysis_str =str(analysis)
        print (symbol + analysis_str)
        self.symbolrec=analysis['RECOMMENDATION']
        print (self.symbolrec)
        dt = time.gmtime()
        ts = calendar.timegm(dt)
        ts_str =str(ts)

        x=self.symbol+self.timeframe+ts_str

        db = firestore.client()  # db e baglantı

        document = db.collection(self.symbol).document(x)
        docId = document.id
        document.set({
            "id": ts_str,

            "position": self.symbolrec,

        })

coin2 = Macdema('ETHUSDT', "1d", 0.015, 1.02, 0.98)
coin1 = Macdema('BTCUSDT', "1d", 0.001, 1.02, 0.98)
coin3 = Macdema('ATOMUSDT', "1d", 2, 1.02, 0.98)
coin4 = Macdema('EOSUSDT', "1d", 20, 1.02, 0.98)
coin5 = Macdema('LITUSDT', "1d", 40, 1.02, 0.98)
coin6 = Macdema('BNBUSDT', "1d", 0.1, 1.02, 0.98)

coin7 = Macdema('CRVUSDT', "1d", 22, 1.02, 0.98)
coin8 = Macdema('THETAUSDT', "1d", 20, 1.02, 0.98)
coin9 = Macdema('XRPUSDT', "1d", 60, 1.02, 0.98)
coin10 = Macdema('RUNEUSDT', "1d", 10, 1.02, 0.98)
coin11 = Macdema('ARUSDT', "1d", 2, 1.02, 0.98)
coin12 = Macdema('DOTUSDT', "1d", 3, 1.02, 0.98)
coin13 = Macdema('ETCUSDT', "1d", 1.5, 1.02, 0.98)
coin14 = Macdema('ALGOUSDT', "1d", 80, 1.02, 0.98)
coin15 = Macdema('TRXUSDT', "1d", 200, 1.02, 0.98)
coin16 = Macdema('LRCUSDT', "1d", 50, 1.02, 0.98)
coin17 = Macdema('SANDUSDT', "1d", 25, 1.02, 0.98)
coin18 = Macdema('AVAXUSDT', "1d", 1, 1.02, 0.98)
coin19 = Macdema('COTIUSDT', "1d", 220, 1.02, 0.98)
coin20 = Macdema('LINKUSDT', "1d", 4, 1.02, 0.98)
coin21 = Macdema('NEARUSDT', "1d", 9, 1.02, 0.98)

coin24 = Macdema('MATICUSDT', "1d", 40, 1.02, 0.98)
coin25 = Macdema('BELUSDT', "1d", 30, 1.02, 0.98)
coin26 = Macdema('KNCUSDT', "1d", 20, 1.02, 0.98)
coin27 = Macdema('BLZUSDT', "1d", 200, 1.02, 0.98)
coin28 = Macdema('KAVAUSDT', "1d", 15, 1.02, 0.98)
coin29 = Macdema('CHZUSDT', "1d", 250, 1.02, 0.98)

coin31 = Macdema('QTUMUSDT', "1d", 8, 1.02, 0.98)
coin32 = Macdema('CHRUSDT', "1d", 150, 1.02, 0.98)
coin33 = Macdema('OCEANUSDT', "1d", 100, 1.02, 0.98)
coin34 = Macdema('ALPHAUSDT', "1d", 150,1.02, 0.98)

while True:
    coin2.dfall('ETHUSDT', "1d")
    time.sleep(15)
    coin1.dfall('BTCUSDT', "1d")
    time.sleep(15)
    coin3.dfall('ATOMUSDT', "1d")
    time.sleep(15)
    coin4.dfall('EOSUSDT', "1d")
    time.sleep(15)
    coin5.dfall('LITUSDT', "1d")
    time.sleep(15)
    coin6.dfall('BNBUSDT', "1d")
    time.sleep(15)
    coin7.dfall('CRVUSDT', "1d")
    time.sleep(15)
    coin8.dfall('THETAUSDT', "1d")
    time.sleep(15)
    coin9.dfall('XRPUSDT', "1d")
    time.sleep(15)
    coin10.dfall('RUNEUSDT', "1d")
    time.sleep(15)
    coin11.dfall('ARUSDT', "1d")
    time.sleep(15)
    coin12.dfall('DOTUSDT', "1d")
    time.sleep(15)
    coin13.dfall('ETCUSDT', "1d")
    time.sleep(15)
    coin14.dfall('ALGOUSDT', "1d")
    time.sleep(15)
    coin15.dfall('TRXUSDT', "1d")
    time.sleep(15)
    coin16.dfall('LRCUSDT', "1d")
    time.sleep(15)
    coin17.dfall('SANDUSDT', "1d")
    time.sleep(15)
    coin18.dfall('AVAXUSDT', "1d")
    time.sleep(15)
    coin19.dfall('COTIUSDT', "1d")
    time.sleep(15)
    coin20.dfall('LINKUSDT', "1d")
    time.sleep(15)
    coin21.dfall('NEARUSDT', "1d")
    time.sleep(15)

    coin24.dfall('MATICUSDT', "1d")
    time.sleep(15)
    coin25.dfall('BELUSDT', "1d")
    time.sleep(15)
    coin26.dfall('KNCUSDT', "1d")
    time.sleep(15)
    coin27.dfall('BLZUSDT', "1d")
    time.sleep(15)
    coin28.dfall('KAVAUSDT', "1d")
    time.sleep(15)
    coin29.dfall('CHZUSDT', "1d")
    time.sleep(15)

    coin31.dfall('QTUMUSDT', "1d")
    time.sleep(15)
    coin32.dfall('CHRUSDT', "1d")
    time.sleep(15)
    coin33.dfall('OCEANUSDT', "1d")
    time.sleep(15)
    coin34.dfall('ALPHAUSDT', "1d")
    time.sleep(15)

    time.sleep(4800)


import json
''' Here this 'Json Object'(inside Serialization.txt) looks like a dictionary of python but it's not
because bollean value true and false are in small letters ( True and False
is written in Python) and null is written,However in python it is written as 'None'.
It is json object therfore it is not written here in .py file.
'''

# The Json Object 'movie info' is inside Serialization.txt
'''Here 'person_info' is a python dictionary
as you can see True , False and None in it.So,
it can be complied by python interpreter
'''
person_info= {
   'name': 'Jhon',
    'age': 30,
    'married': True,
    'divorced': False,
    'children': ('Ann', 'Billy'),
    'pets': None,
     'cats': [
         {'model': 'BMW 230', 'mpg': 27.5},
         {'model': 'Ford Edge','mpg': 24.1}
         ]
    }

with open('D:\Visual Studio 2017\Repos\PythonApplication3\PythonApplication3\Deserialization.txt','r+',encoding='utf-8') as f:
    movie_data = json.load(f) #converts json object into python dict and displays output
   # person_data= json.dump(person_info,f,indent=4, sort_keys= True) #converts Python dict into Json object and stores in file

print(movie_data["title"]) #you can even access the data by key

with open('D:\Visual Studio 2017\Repos\PythonApplication3\PythonApplication3\Serialization.txt','r+') as f:
    person_data= json.dump(person_info,f, indent=4,sort_keys=True) #dumped into Serialization.txt file
    pointer_position=f.tell()
    #sot_keys specifies if the result should be sorted or not

    #if you want dump in ceratain position you run the program then use seek() and tell()

print(pointer_position)
print('Here is the output of load() ,\n which is Python dicionary.Initially it was Json Object')
print(movie_data)
print(type(movie_data))

print('here is the output of dump(),\n here Python Dict is taken to file \n stored as Json object')
print(person_data)
print('It\'s type when it was written in Python', type(person_info), 'and now it is converted to JSON Object.')
print(type(person_data))


from django.shortcuts import render
from .serializers import *
from rest_framework import generics, permissions, status, viewsets
from .models import *
import requests
import json
from django.http import HttpResponse
from rest_framework_simplejwt.views import TokenObtainPairView
from rest_framework.response import Response
from rest_framework.views import APIView

class CustomUserCreate(APIView):
    permission_classes = (permissions.AllowAny,)

    def post(self, request, format='json'):
        serializer = CustomUserSerializer(data=request.data)
        if serializer.is_valid():
            user = serializer.save()
            if user:
                json = serializer.data
                return Response(json, status=status.HTTP_201_CREATED)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class RetrieveOutingView(generics.ListAPIView):
    queryset = Outing.objects.all()
    serializer_class = OutingSerializer

def YelpView(request):

    city = request.GET.get('city')
    lat = request.GET.get('lat')
    lng = request.GET.get('lng')
    if city:
        print('city sent')
        queryParam = f"location={city}"
    else:
        print('something else sent')
        queryParam = f"latitude={lat}&longitude={lng}"
    price = request.GET.get('price')
    term = f"term={request.GET.get('term')}"
    url = f"https://api.yelp.com/v3/businesses/search?{queryParam}&{term}&categories=&price={price}&sort_by=best_match&matches_party_size_param=true&limit=5"

    headers = {
        "accept": "application/json",
        "Authorization": "Bearer MirzBJCP0rFnEzwjGZKMhO8tiXa3oyOeJO7JpCIr7ExtZAmAmScyj4wjuMlrR28Msy97mfUV7GDnEXLNak8-gSn8ReYrVhTj9vrrgrgjM86SnIuXeCUytkvY1nOHY3Yx"
    }
    response = requests.get(url, headers=headers)
    data = response.json()
    return HttpResponse(json.dumps(data), content_type="application/json")

class RetrieveActivityView(viewsets.ModelViewSet):
    queryset = Activity.objects.all()
    serializer_class = ActivitySerializer

    def get_queryset(self):
        username = self.request.user
        return Activity.objects.filter(user=username)

class RetrieveUserActivityView(generics.RetrieveUpdateDestroyAPIView):
    queryset = UserActivity.objects.all()
    serializer_class = UserActivitySerializer


import matplotlib.pyplot as plt
from pylab import mpl

# 用以显示中文
mpl.rcParams['font.sans-serif'] = ['FangSong']

decisionNode = dict(boxstyle="sawtooth", fc="0.8")
leafNode = dict(boxstyle="round4", fc="0.8")
arrow_args = dict(arrowstyle="<-")

def plotNode(nodeTxt, centerPt, parentPt, nodeType):
    createPlot.axl.annotate(nodeTxt, xy=parentPt, xycoords='axes fraction', xytext=centerPt, textcoords='axes fraction',
                            va="center", ha="center", bbox=nodeType, arrowprops=arrow_args)

def createPlot():
        fig = plt.figure(1, facecolor='white')
        fig.clf()
        createPlot.axl = plt.subplot(111, frameon=True)
        plotNode('决策节点', (0.5, 0.1), (0.1, 0.5), decisionNode)
        plotNode('叶节点', (0.8, 0.1), (0.3,0.8), leafNode)
        plt.show()

def getNumLeafs(myTree):
    numLeafs = 0
    # in the py3, type(myTree.keys()) is dict_keys. It can be directed used for iteration, but index is not suitable.
    # in the py2, type(myTree.keys()) is list.
    firstStr = list(myTree.keys())[0]
    secondDict = myTree[firstStr]
    for key in secondDict.keys():
        if type(secondDict[key]).__name__ == 'dict':
            numLeafs += getNumLeafs(secondDict[key])
        else:
            numLeafs += 1
    return numLeafs

def getTreeDepth(myTree):
    maxDepth = 0
    firstStr = list(myTree.keys())[0]
    secondDict = myTree[firstStr]
    for key in secondDict.keys():
        if type(secondDict[key]).__name__ == 'dict':
            thisDepth = 1 + getTreeDepth(secondDict[key])
        else:
            thisDepth = 1
        if thisDepth > maxDepth:
            maxDepth = thisDepth
    return maxDepth

def retrieveTree(i):
    listOfTrees = [{'no surfacing': {0: 'no', 1: {'flippers': {0: 'no', 1: 'yes'}}}},
                   {'no surfacing': {0: 'no', 1: {'flippers': {0: {'head': {0: 'no', 1: 'yes'}}, 1: 'no'}}}}
                   ]
    return listOfTrees[i]

def plotMidText(cntrPt, parentPt, txtString):
    xMid = (parentPt[0] - cntrPt[0])/2.0 + cntrPt[0]
    yMid = (parentPt[1] - cntrPt[1])/2.0 + cntrPt[1]
    createPlot.axl.text(xMid, yMid, txtString)

def plotTree(myTree, parentPt, nodeTxt):
    numLeafs = getNumLeafs(myTree)
    depth = getTreeDepth(myTree)
    firstStr = list(myTree.keys())[0]
    cntrPt = (plotTree.xOff + (1.0 + float(numLeafs))/2.0/plotTree.totalW, plotTree.yOff)
    plotMidText(cntrPt, parentPt, nodeTxt)
    plotNode(firstStr, cntrPt, parentPt, decisionNode)
    secondDict = myTree[firstStr]
    plotTree.yOff = plotTree.yOff - 1.0/plotTree.totalD
    for key in secondDict.keys():
        if type(secondDict[key]).__name__ == 'dict':
            plotTree(secondDict[key], cntrPt, str(key))
        else:
            plotTree.xOff = plotTree.xOff + 1.0/plotTree.totalW
            plotNode(secondDict[key], (plotTree.xOff, plotTree.yOff),cntrPt, leafNode)
            plotMidText((plotTree.xOff, plotTree.yOff), cntrPt, str(key))
    plotTree.yOff = plotTree.yOff + 1.0/plotTree.totalD

def createPlot(inTree):
    fig = plt.figure(1, facecolor='white')
    fig.clf()
    axprops = dict(xticks=[], yticks=[])
    createPlot.axl = plt.subplot(111, frameon=False, **axprops)
    plotTree.totalW = float(getNumLeafs(inTree))
    plotTree.totalD = float(getTreeDepth(inTree))
    plotTree.xOff = -0.5/plotTree.totalW
    plotTree.yOff = 1.0
    plotTree(inTree, (0.5, 1.0), '')
    plt.show()

# myTree = retrieveTree(0)
# myTree['no surfacing'][3] = 'maybe'
# createPlot(myTree)


#! usr/bin/python
# -*- coding: utf-8 -*-
"""
This script outputs your Deveropment Environment Version(Deever)
to Markdown file.

Usage:
  toolsver [-h] [-o=<output>]

Options:
  -h --help    show this
  -o=<output>  named output file [default: ./ENVIRONMENT.md]
"""

import pip
import platform
from datetime import datetime
from collections import OrderedDict
from docopt import docopt

class Deever():
    def __init__(self, outp):
        self.outp = str(outp)
        self.freeze = pip.operations.freeze.freeze

    def get_environment(self):
        return [
            datetime.now().isoformat(" ").lstrip(),
            "Python " + platform.python_version(),
            "Compiler: " + platform.python_compiler(),
            "Implementation: " + platform.python_implementation()
        ]

    def get_os_version(self):
        return [platform.version(), platform.processor()]

    def get_modules_version(self):
        return [_ for _ in self.freeze()]

    def set_environment(self):
        return OrderedDict([
            ("# ENVIRONMENT", self.get_environment()),
            ("## OS", self.get_os_version()),
            ("## MODULES", self.get_modules_version()),
        ])

    def write_environment(self):
        with open(self.outp, "w") as f:
            for k, vs in self.set_environment().items():
                f.write("{0}".format(k) + "  \n")
                for v in vs:
                    v = v.rstrip("\n") + "  \n"
                    f.write("{0}".format(v))

def main(outp):
    D = Deever(outp)
    D.write_environment()
    return "Everything is OK"

if __name__ == '__main__':
    args = docopt(__doc__, version='0.1.0')
    print(main(args["-o"]))


#!/usr/bin/python3

# methods for feature computation and alignment at the frame, segment, and
# utterance levels.

import pandas as pd
import numpy as np
from itertools import repeat

from .segmentaxis import segment_axis as sa
from ..utils.utils import apply_parallel

# calculate delta and delta-delta features in an utterance dataframe
# delta computation modified from https://github.com/jameslyons/
# python_speech_features/blob/master/python_speech_features/base.py
def roll_delta(dfg, n=2):
    w = np.arange(-n, n + 1)
    d = 1. / np.sum([2 * i * i for i in range(1, n + 1)])

    # uses segment_axis to efficiently generate an overlapping window view of
    # the data and apply the weighting/summing delta function
    def delta(nd):
        pad0 = [nd[0] for _ in range(n)]
        pad1 = [nd[-1] for _ in range(n)]
        padded = np.concatenate((pad0, nd, pad1))
        return np.sum(sa(pad, n * 2 + 1, n * 2, 0).swapaxes(1, 2) * w, axis=-1) * d

    nd_d = delta(dfg.values)
    nd_dd = delta(nd_d)

    return pd.concat((dfg,
                      pd.DataFrame(nd_d, index=dfg.index, dtype=np.float16),
                      pd.DataFrame(nd_dd, index=dfg.index, dtype=np.float16)), axis=1)

# computes delta features in parallel

def compute_deltas(df_mfcc, n=2):
    # group features by utterance and compute deltas/delta-deltas
    mg = df_mfcc.groupby(df_mfcc.index)
    df_deltas = apply_parallel(roll_delta, [(g, n) for _, g in mg])

    # fix column names
    c_ = [c for c in df_mfcc.columns]
    c_d = ['d_' + c for c in df_mfcc.columns]
    c_dd = ['dd_' + c for c in df_mfcc.columns]
    df_deltas.columns = c_ + c_d + c_dd
    return df_deltas

# aligns MFCC frames in one dataframe to phone segments in another dataframe
# creates a new column in the MFCC dataframe indicating phone count, i.e. the
# phone order of a particular frame within an utterance
# TODO the MFCC data doesn't actually need to be passed to this function
# NOTE actually it does because otherwise you'll need to concat it afterwards
def count_phones_per_spkr(dfg_mfcc, dfg_phon):
    spk_frames = []
    for utt, frames in dfg_mfcc.groupby(dfg_mfcc.index):
        seg = dfg_phon.loc[utt]
        idx = [(i, r) for i in range(len(seg))
               for r in repeat(seg.phon.iloc[i], seg.dur.iloc[i])]
        dif = len(frames) - len(idx)
        if dif > 0:
            idx.extend([idx[-1] for _ in range(dif)])
        spk_frames.extend(idx)
    spk_frames = np.array(spk_frames, dtype=np.uint16)
    return dfg_mfcc.assign(ord=spk_frames[:, 0], phon=spk_frames[:, 1])

# align MFCC dataframe to phone dataframe.
# returns frame-level alignments for phone segment number (within utterance),
# phone symbol, and speaker.

def align_phones(df_mfcc, df_phon, spk_group=(), seq=False):
    # use the minimal subset of utt/files contained in both dataframes
    # drop utterances with only 1 phone
    dif = set(df_mfcc.index) ^ set(df_phon.index)
    drop = set(df_phon.loc[df_phon.groupby(
        df_phon.index).dur.count() == 1].index)
    drop |= dif
    df_mfcc.drop(drop, inplace=True)
    df_phon.drop(drop, inplace=True)

    # speaker information should be encoded somehow in the utterance filename
    # the grouper is the pattern needed to extract this information
    # in this case, the speaker can be identified by the first 9 chars in file
    if spk_group:
        start, end = spk_group
        dfg_spk_mfcc = df_mfcc.groupby(df_mfcc.index.str[start: end])
        dfg_spk_phon = df_phon.groupby(df_phon.index.str[start: end])
        args = [(g, dfg_spk_phon.get_group(n)) for n, g in dfg_spk_mfcc]
    else:
        dfg_mfcc = df_mfcc.groupby(df_mfcc.index)
        dfg_phon = df_phon.groupby(df_phon.index)
        args = [(g, dfg_phon.get_group(n)) for n, g in dfg_mfcc]

    if seq:
        ali = pd.concat([count_phones_per_spkr(m, p) for m, p in args])
    else:
        ali = apply_parallel(count_phones_per_spkr, args)
    return ali

def calc_segs(spk):
    dfg = spk.groupby([spk.index, spk.ord])

    dur = pd.Series(dfg.eng.count(), name='dur', dtype=np.int16)
    d_dur = dur.groupby(dur.index.get_level_values(0)).diff()
    dd_dur = d_dur.groupby(d_dur.index.get_level_values(0)).diff()
    d_dur = d_dur.fillna(0).astype(np.int16)
    dd_dur = dd_dur.fillna(0).astype(np.int16)
    d_dur.name = 'd_dur'
    dd_dur.name = 'dd_dur'

    mean = dfg.mean()
    phon = mean.phon
    mean.drop('phon', axis=1, inplace=True)
    d_mean = mean.groupby(mean.index.get_level_values(0)).diff()
    dd_mean = d_mean.groupby(d_mean.index.get_level_values(0)).diff()
    d_mean = d_mean.fillna(0).astype(np.float16)
    dd_mean = dd_mean.fillna(0).astype(np.float16)

    mean.columns = ['avg_' + c for c in mean.columns]
    d_mean.columns = ['d_avg_' + c for c in d_mean.columns]
    dd_mean.columns = ['dd_avg_' + c for c in dd_mean.columns]

    return pd.concat((phon, dur, mean, d_dur, d_mean, dd_dur, dd_mean), axis=1)

# compute segment-level features for utterance classification from a phone
# aligned mfcc dataframe. features include per segment frame averages, variances
# and their deltas.
def compute_seg_feats(df_ali, spk_group=(0, 9)):
    if spk_group:
        start, end = spk_group
        args = [[g] for _, g in df_ali.groupby(df_ali.index.str[start:end])]
    else:
        arts = [[g] for _, g in df_ali.groupby(df_ali.index)]

    return apply_parallel(calc_segs, args)

def compute_utt_feats(df_seg):
    """
    df_seg = df_seg.drop(
        df_seg.columns[df_seg.columns.str.contains('phon')], axis=1)
    dfg_feat = df_seg.groupby(df_seg.index.get_level_values(0))
    df_utt = pd.concat((dfg_feat.mean().astype(np.float16),
                       dfg_feat.var().astype(np.float16)), axis=1)
    cols = df_seg.columns
    df_utt.columns = ['avg_'+c for c in cols] + ['var_'+c for c in cols]
    """
    # drop silence segments at the beginning and end of utterances
    df = df_seg.drop(df_seg.loc[df_seg.phon == 1].index)
    # drop phone column
    df = df.drop('phon', axis=1)
    # group by utterance
    dfg_utt = df.groupby(df.index.get_level_values(0))
    # compute normalized means
    df_utt = dfg_utt.mean() / dfg_utt.std()
    # drop utterances with NaN or Inf
    df_utt = df_utt[np.all(np.isfinite(df_utt), axis=1)]

    return df_utt

def compute_vad(feats, context=5, proportion=0.8):
    # assumes first column of feats is log energy
    log_eng = feats[:, 0]
    # simple energy based cutoff, needs adjustment for real time usage
    cutoff = 5 + 0.5 * log_eng.mean()
    # within a context window centred around the current frame, count the
    # number of frames above the cutoff point
    props = np.array([np.sum(win > cutoff) for win in sa(np.concatenate(
        (np.zeros(context), log_eng, np.zeros(context))), context * 2 + 1, context * 2)])
    # return True (voiced) if the number of frames above the cutoff is above a
    # certain proportion
    return props / context > proportion


import decimal
import json
import logging
import re
import time
import uuid
from decimal import Decimal
from urllib.parse import urlparse, urlunparse
from threading import Thread
from django.contrib import messages
from django.contrib.sites.models import Site
from asgiref.sync import async_to_sync
from channels.layers import get_channel_layer

logger = logging.getLogger(__name__)

def urlEncodeNonAscii(b):
    return re.sub(b'[\x80-\xFF]', lambda c: ('%%%02x' % ord(c.group(0))).encode('ascii'), b)

def iriToUri(iri):
    parts = urlparse(iri)
    return urlunparse([
        part.encode('idna') if parti==1 else urlEncodeNonAscii(part.encode('utf-8'))
        for parti, part in enumerate(parts)
    ])

class ChallengeUpdateThread(Thread):
    def __init__(self, course):
        Thread.__init__(self)
        self.course = course

    def run(self):
        for challenge in self.course.challenges.all():
            challenge.compute_course(self.course)
            challenge.compute_challenge()
            #messages.add_message(request, messages.INFO, u'Classement du challenge "%s" calculé' % (challenge.nom, ))

class ChallengeInscriptionEquipe(Thread):
    def __init__(self, equipe):
        Thread.__init__(self)
        self.equipe = equipe

    def run(self):
        time.sleep(10)

        for challenge in self.equipe.course.challenges.all():
            try:
                challenge.inscription_equipe(self.equipe)
            except:
                import traceback
                traceback.print_exc()
                logger.exception('error updating challenge')

def jsonDate(obj):
    if hasattr(obj, 'isoformat'):
        return obj.isoformat()
    else:
        raise TypeError('Object of type %s with value of %s is not JSON serializable' % (type(obj), repr(obj)))

def floor(value):
    with decimal.localcontext() as ctx:
        ctx.rounding = decimal.ROUND_FLOOR
        return value.quantize(Decimal('1.00'))

def roundcent(value):
    if value - floor(value) > Decimal('0.005'):
        return Decimal('0.01')
    return Decimal('0.00')

def round(value):
    return floor(value) + roundcent(value)

def repartition_frais(montants, frais):
    """
    Calcul de la repartition des frais au prorata avec arrondis au centime
    Assure que le total des arrondis est bien equal au total des frais
    quite a ajouter un centime en plus si besoin
    """
    total = Decimal('0.00')
    for montant in montants:
        total += montant

    frais_list = []
    montant_arrondis = Decimal('0.00')
    for montant in montants:
        montant_frais = montant / total * frais
        frais_list.append([
            floor(montant_frais),
            roundcent(montant_frais)
        ])
        montant_arrondis += floor(montant_frais)
    for item in frais_list:
        if item[1] == Decimal('0.01') and montant_arrondis < frais:
            item[0] += item[1]
            montant_arrondis += item[1]
    for item in frais_list:
        if item[1] == Decimal('0') and montant_arrondis < frais:
            item[0] += Decimal('0.01')
            montant_arrondis += Decimal('0.01')
    return (item[0] for item in frais_list)

def send_mail(**kwargs):
    channel_layer = get_channel_layer()
    message = {
        'type': 'send.mail',
    }
    message.update(kwargs)
    if 'message_id' not in message:
        message['message_id'] = '%s@%s' % (uuid.uuid4().hex, Site.objects.get_current())
    #message['to'] = json.dumps(message['to'])
    async_to_sync(channel_layer.send)("mail", message)


# -*- coding: utf-8 -*-
import csv
import time

from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait

from load_driver import *

driver = load_firefox()
# driver = load_chrome()

csv_file = 'comments.csv'
try:
    driver.get("https://www.bilibili.com/video/av22957730")
    # 滚动到 comment 元素
    target = driver.find_element_by_class_name("comment")
    driver.execute_script("arguments[0].scrollIntoView();", target)
    print(driver.title)
    # 写入 CSV 标题
    with open(csv_file, 'w', newline='', encoding='utf-8-sig') as f:
        writer = csv.writer(f, dialect='excel')
        writer.writerow(['楼层', '来源', '评论时间', '用户名', '评论内容'])

    # 获取所有评论
    while True:
        wait = WebDriverWait(driver, 10)
        wait.until(EC.presence_of_element_located((By.CLASS_NAME, 'list-item')))
        comments = target.find_elements_by_xpath('.//div[@class="comment-list"]/div[contains(@class,"list-item")]')
        print('comments count', len(comments))
        for comment in comments:
            # 楼层
            floor = comment.find_element_by_xpath('./div[2]/div[@class="info"]/span[@class="floor"]').text
            # 评论来源（来自安卓客户端 or ...）
            plad = ''
            try:
                plad = comment.find_element_by_xpath('./div[2]/div[@class="info"]/span[@class="plad"]').text
            except:
                pass
            # 评论时间
            comment_time = comment.find_element_by_xpath('./div[2]/div[@class="info"]/span[@class="time"]').text
            # 用户名
            username = comment.find_element_by_xpath('./div[2]/div[@class="user"]/a[1]').text
            # 评论内容
            text = comment.find_element_by_xpath('./div[2]/p[@class="text"]').text
            print('floor', floor)
            print('plad', plad)
            print('comment_time', comment_time)
            print('username', username)
            print('text', text)
            row = [floor, plad, comment_time, username, text]
            with open(csv_file, 'a', newline='', encoding='utf-8-sig') as f:
                writer = csv.writer(f, dialect='excel')
                writer.writerow(row)
        try:
            # 点击 下一页
            driver.find_element_by_link_text('下一页').click()
            time.sleep(3)
        except:
            print('没有下一页了')
            break

    time.sleep(5)
except Exception as e:
    print(e)
finally:
    driver.quit()
    pass


from sklearn.preprocessing import MinMaxScaler
from sklearn.datasets import load_iris
import pandas as pd

iris = load_iris()
iris_data = iris.data
iris_df = pd.DataFrame(data=iris_data,columns=iris.feature_names)

# MinMaxScaler : 데이터값을 0과 1사이의 범위 값으로 변환합니다(음수 값이 있으면 -1 에서 1값으로 변환합니다.)
scaler = MinMaxScaler()
scaler.fit(iris_df)
iris_scaled = scaler.transform(iris_df)

iris_df_scaled = pd.DataFrame(data=iris_scaled, columns=iris.feature_names)
print("feature들의 최소값")
print(iris_df_scaled.min())
print("feature들의 최대값")
print(iris_df_scaled.max())

# print(iris_df_scaled)


import threading
import time
# def test(x,y):
#     for i in range(x,y):
#         print(i)
#
# thread1 = threading.Thread(name='t1',target=test,args=(1,10))
# thread2 = threading.Thread(name='t2',target=test,args=(11,20))
# thread1.start()
# thread2.start()
#
# class mythread(threading.Thread):
#     def run(self):
#         for i in range(1,10):
#             print(i)
# thread1 = mythread()
# thread2 = mythread()
# thread1.start()
# thread2.start()

# def test():
#     time.sleep
#     for i in range(10):
#         print(i)
# thread1 = threading.Thread(target=test,daemon=False)
# thread1.start()
# thread1.join()
# print('主线程完成了')

import time
import requests
from multiprocessing.dummy import Pool

# 自定义函数
def query(url):
    requests.get(url)

start = time.time()
for i in range(100):
    query('https://www.csdn.net/')
end = time.time()
print(f'单线程循环访问100次CSDN，耗时：{end - start}')

start = time.time()
url_list = []
for i in range(100):
    url_list.append('https://www.csdn.net/')
pool = Pool(5)
pool.map(query, url_list)
end = time.time()
print(f'5线程访问100次CSDN，耗时：{end - start}')


import os
import glob
import torch
import torch.nn
import torch.utils.data
import torchvision
from PIL import Image

from config_val import get_config
import numpy as np
import res2net2
import math

def trans_img(img_path, data_transform, device):
    ori_image = data_transform(Image.open(str(img_path))).unsqueeze(0)
    ori_image = ori_image.to(device)
    ori_image = ori_image[:,:3,:,:]
    return ori_image

def make_test_data(cfg, img_path_list, device):
    data_transform = torchvision.transforms.Compose([
        torchvision.transforms.ToTensor(),

    ])
    imgs = []
    for img_path in img_path_list:

        depth_name = img_path.replace('png', 'npy')

        guide_image_name = img_path.replace('input', 'guide')
        guide_depth_name = depth_name.replace('input', 'guide')

        ori_image = trans_img(img_path, data_transform, device)
        guide_image = trans_img(guide_image_name, data_transform, device)

        depth =  np.load(depth_name, allow_pickle=True).item()['normalized_depth']
        ori_depth = torch.unsqueeze(torch.from_numpy(depth), 0)
        ori_depth = torch.unsqueeze(ori_depth, 0)

        guide_depth = np.load(guide_depth_name, allow_pickle=True).item()['normalized_depth']
        guide_depth = torch.unsqueeze(torch.from_numpy(guide_depth), 0)
        guide_depth = torch.unsqueeze(guide_depth, 0)

        imgs.append([ori_image, guide_image, ori_depth, guide_depth])

    return imgs

def main(cfg):
    # -------------------------------------------------------------------
    # basic config
    print(cfg)
    if cfg.gpu > -1:
        os.environ['CUDA_VISIBLE_DEVICES'] = str(cfg.gpu)
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    # -------------------------------------------------------------------

    path = cfg.input_data_path
    if cfg.gt:
        gt_path = cfg.ori_data_path
    output_dir = cfg.output_dir

    name = os.listdir(path)
    print('Start eval')
    networks_description = cfg.model_path
    network = res2net2.Dehaze3().cuda()
    network.load_state_dict(torch.load(networks_description)['state_dict'])

    networks = [network]

    if not os.path.exists(os.path.join(output_dir, cfg.name)):
        os.makedirs(os.path.join(output_dir, cfg.name))

    if cfg.gt:
        psrn, ssim = [], []

    for i in name:

        if i.endswith('.npy'):
            continue

        test_file_path = os.path.join(path, i)
        test_file_path = glob.glob(test_file_path)
        ori_image, guide_image, ori_depth, guide_depth = make_test_data(cfg, test_file_path, device)[0]
        ori_image, guide_image, ori_depth, guide_depth = ori_image.cuda(), guide_image.cuda(), ori_depth.cuda(), guide_depth.cuda()

        for idx, network in enumerate(networks):

            network.cuda().eval()

            if idx == 0:

                dehaze_image = network(ori_image, guide_image, ori_depth, guide_depth)
                dehaze_image = dehaze_image.cpu()
            else:
                tmp =  network(test_img.cuda(), test_depth.cuda()).detach()
                dehaze_image += tmp.cpu()

            network.cpu()

        dehaze_image = dehaze_image/(len(networks))
        dehaze_image = dehaze_image.cuda()

        torchvision.utils.save_image(dehaze_image, os.path.join(output_dir, cfg.name, i))

        del dehaze_image

    if cfg.gt:
        print(sum(psrn)/len(psrn), sum(ssim)/len(ssim))

if __name__ == '__main__':
    config_args, unparsed_args = get_config()
    main(config_args)


#!/usr/bin/env python3

from time import sleep
from random import uniform
import os
import openai

openai.api_key = os.environ['GPT_KEY']

prompt = input("Enter your prompt \n -> ")

completion = openai.ChatCompletion.create(
  model="gpt-3.5-turbo",
  messages=[
    {"role": "user", "content": prompt},
  ]
)

response = completion.choices[0].message.content

for x in response:
    print(x, end='', flush=True)
    sleep(uniform(0, 0.1))


from modules.p2pNetwork.messaging.MessageQueue import MessageQueue, Task
from modules.state.variables.BlockChain import BlockChain
from modules.state.variables.LoggedInUser import LoggedInUser
from modules.state.variables.SelectedBlock import SelectedBlock
from modules.blockchain.TransactionBlock import TransactionBlock
from modules.view.actions.IAction import IAction
import State

class ValidateBlockAction(IAction):

    def __init__(self, page, di_container):
        super().__init__(page, di_container)

    def handle(self):
        block : TransactionBlock = State.instance(SelectedBlock).get_value()
        if block is None:
            raise ValueError('Block which is None cannot be validated. Please make sure to have selected a block.')
        if State.instance(LoggedInUser).get_value() is None:
            raise ValueError('You need to be logged in to validate a block. Please go back to the main menu and log in.')

        block.validate(State.instance(LoggedInUser).get_value().public_key)
        chain : TransactionBlock = State.instance(BlockChain).get_value()
        self.update_chain(block, chain)
        State.instance(BlockChain).set_value(chain)
        queue : MessageQueue = State.instance(MessageQueue).get_value()
        task = Task(("CLIENT", "BLOCKCHAIN_UPDATE"), chain)
        queue.lock()
        queue.enqueue(task)
        queue.release()
        return self.page.options.get('1')

    def update_chain(self, block: TransactionBlock, chain: TransactionBlock):
        if chain.blockId == block.blockId:
                chain.validated_by = block.validated_by
        if chain.previousBlock is not None:
            return self.update_chain(block, chain.previousBlock)
        return chain


'''
Created on 9 Aug 2020

@author: czarny
'''

import datetime
import calendar
import os
from workalendar.europe import Ireland

dateOfPayment = "2021/5/6"
#dateOfPayment = "2020/3/17"
dateOfPayment = "2020/12/28"

irishHolidays = [(datetime.date(2020, 1, 1), 'New year'),
                (datetime.date(2020, 3, 17), "Saint Patrick's Day"),
                (datetime.date(2020, 4, 13), 'Easter Monday'),
                (datetime.date(2020, 5, 4), 'May Day'),
                (datetime.date(2020, 5, 5), 'May Day1'),
                (datetime.date(2020, 5, 6), 'May Day2'),
                (datetime.date(2020, 6, 1), 'June Holiday'),
                (datetime.date(2020, 8, 3), 'August Holiday'),
                (datetime.date(2020, 10, 26), 'October Holiday'),
                (datetime.date(2020, 12, 25), 'Christmas Day'),
                (datetime.date(2020, 12, 26), "St. Stephen's Day"),
                (datetime.date(2020, 12, 28), "St. Stephen's Day Shift")]

# month number is extracted from data
d = datetime.datetime.strptime(dateOfPayment, '%Y/%m/%d')
paydate = datetime.date(d.year,d.month,d.day)

# debugging
for d in irishHolidays:
        print(d)

# bank holiday
def isBankHolidayFunc(date):
    for d in Ireland().holidays(date.year):
    #for d in irishHolidays:
        if d[0] == date: return True
    return False

#print(isBankHolidayFunc(paydate))

# weekend day check
def isWeekendFunc(date):
    if date.weekday() > 5: return True
    return False

#print(isWeekendFunc(paydate))

def previousWorkingDay(date):
    return date - datetime.timedelta(max(1, (date.weekday() + 6) % 7 - 3))

if isWeekendFunc(paydate):
    print("this is weekend early date needed")
    newPaydate = previousWorkingDay(paydate)
    while isBankHolidayFunc(newPaydate):
        newPaydate = previousWorkingDay(newPaydate)
else:
    print("this is week day")
    #newPaydate = previousWorkingDay(paydate)
    while isBankHolidayFunc(paydate):
        paydate = previousWorkingDay(paydate)
        print("changed day ", paydate)
    newPaydate = paydate
print(newPaydate)

'''
# bank holiday
isBankHoliday = False
def isBankHolidayFunc(date, isBaHol):
    print(date.year)
    for d in Ireland().holidays(date.year):
        if d[0] == date:
            isBaHol = True
            print("to jest bank holiday ", d[0]) # debugging
    return isBaHol

isBankHoliday = isBankHolidayFunc(paydate, isBankHoliday)
print(isBankHoliday)

# weekend day check
isWeekend = False

def isWeekendFunc(date,isWeEnd):
    #weekday = date.weekday()
    if date.weekday() > 5:
        isWeEnd = True
    return isWeEnd

isWeekend = isWeekendFunc(paydate, isWeekend)
print(isWeekend)

# amended working day is generated
def previousWorkingDay(date):
    return date - datetime.timedelta(max(1, (date.weekday() + 6) % 7 - 3))

print(previousWorkingDay(paydate))
'''


import json

from realms_cli.binary_converter import decimal_to_binary, map_realm, map_crypt

# f = open("data/realms_bit.json", "a")
# output = []
# for index in range(8000):
#     output.append({str(index + 1): map_realm(realms[str(index + 1)])})

# f.write(str(output))

# # with open('scripts/json_data.json', 'w') as outfile:
# #     outfile.write(str(create_output(buildings, 6)))

building_costs = [6, 6, 6, 6, 6, 6, 6, 6, 6]

resource_ids = [1, 4, 6]
resource_values = [10, 10, 10, 10, 10]

buildings = [
    {
        "name": "Fairgrounds",
        "id": 1,
        "costs": [2, 12, 31, 21, 7],
        "ids":[2, 2, 3, 4, 7]
    }
]

# Quickly test Realms metadata
realms = json.load(open('data/realms.json'))
resources = json.load(open('data/resources.json'))
orders = json.load(open('data/orders.json'))
wonders = json.load(open('data/wonders.json'))

print(decimal_to_binary(resource_ids, 8))
print(decimal_to_binary(resource_values, 12))

print(map_realm(realms["1"], resources, wonders, orders))

# Quickly test Crypts metadata
crypts = json.load(open("data/crypts.json"))
environments = json.load(open("data/crypts_environments.json"))
affinities = json.load(open("data/crypts_affinities.json"))

print(map_crypt(crypts["1"], environments, affinities))


from zipfile import ZipFile
from xml.dom.minidom import parse, parseString
from isodate import parse_datetime
from io import StringIO

class SazFile:
    class ParseError(Exception):
        pass

    def __init__(self, filename):
        self.filename = filename
        self._zipfile = None
        self._sessions = []
        self._cfilelist = []
        self._mfilelist = []
        self._sfilelist = []
        self._session_num = 0

    @property
    def zipfile(self):
        if self._zipfile is None:
            self._zipfile = ZipFile(self.filename)
        return self._zipfile

    @property
    def cfilelist(self):
        if not self._cfilelist:
            self._cfilelist = [
                x for x in self.zipfile.namelist()
                if x.startswith('raw') and x.endswith('_c.txt')
            ]
        return self._cfilelist

    @property
    def mfilelist(self):
        if not self._mfilelist:
            self._mfilelist = [
                x for x in self.zipfile.namelist()
                if x.startswith('raw') and x.endswith('xml')
            ]
        return self._mfilelist

    @property
    def sfilelist(self):
        self._sfilelist = [
                x for x in self.zipfile.namelist()
                if x.startswith('raw') and x.endswith('_s.txt')
            ]
        return self._sfilelist

    @property
    def html(self):
        fname = '_index.htm'
        return self.zipfile.read(fname).decode('utf-8')

    @property
    def content_type(self):
        pass

    @property
    def session_num(self):
        if self._session_num == 0:
            clen = len(self.cfilelist)
            mlen = len(self.mfilelist)
            slen = len(self.sfilelist)

            if not (clen == mlen == slen):
                raise self.ParseError('files missing')

            self._session_num = clen

        return self._session_num

    @property
    def sessions(self):
        if self._sessions == []:
            for i in range(self.session_num):
                rawdata = {
                    "c": self.zipfile.read(self.cfilelist[i]),
                    "m": self.zipfile.read(self.mfilelist[i]),
                    "s": self.zipfile.read(self.sfilelist[i]),
                }
                session = Session(rawdata)
                self._sessions.append(session)

        return self._sessions

    @property
    def sequence_time(self):
        starttime = min([
            parse_datetime(s.timing['ClientBeginRequest'])
            for s in self.sessions
            if str(parse_datetime(s.timing['ClientBeginRequest'])) != '0001-01-01 00:00:00'
        ])

        endtime = max([
            parse_datetime(s.timing['ClientDoneResponse'])
            for s in self.sessions
            if str(parse_datetime(s.timing['ClientDoneResponse'])) != '0001-01-01 00:00:00'
        ])

        return (endtime - starttime).total_seconds()

class Session:
    def __init__(self, rawdata):
        self._rawdata = rawdata
        self._crequest = None
        self._srequest = None
        self._metadata = None

    @property
    def client_request(self):
        if self._crequest is None:
            self._crequest = ClientRequest(self._rawdata['c'])

        return self._crequest

    @property
    def server_request(self):
        if self._srequest is None:
            self._srequest = ServerRequest(self._rawdata['s'])

        return self._srequest

    @property
    def metadata(self):
        if self._metadata is None:
            self._metadata = MetaData(self._rawdata['m'])

        return self._metadata

    @property
    def timing(self):
        return self.metadata.timing

    @property
    def is_static(self):
        return self.server_request.content_type in [
            b'image/png', b'image/gif', b'text/javascript', b'text/css'
        ]

    @property
    def https_handshake_time(self):
        return int(self.timing['HTTPSHandshakeTime'])

    @property
    def tcp_connec_time(self):
        return int(self.timing['TCPConnectTime'])

    @property
    def dns_time(self):
        return int(self.timing['DNSTime'])

    @property
    def gateway_time(self):
        return int(self.timing['GatewayTime'])

    @property
    def server_time(self):
        ret = 0
        starttime = parse_datetime(self.timing['ServerGotRequest'])
        endtime = parse_datetime(self.timing['ServerBeginResponse'])

        if not (str(starttime) == '0001-01-01 00:00:00' \
           or str(endtime) == '0001-01-01 00:00:00'):
            ret = (endtime - starttime).total_seconds()
            ret = 0 if ret < 0 else ret

        return ret

    @property
    def download_time(self):
        ret = 0
        starttime = parse_datetime(self.timing['ServerBeginResponse'])
        endtime = parse_datetime(self.timing['ClientDoneResponse'])

        if not (str(starttime) == '0001-01-01 00:00:00' \
           or str(endtime) == '0001-01-01 00:00:00'):
            ret = (endtime - starttime).total_seconds()
            ret = 0 if ret < 0 else ret

        return ret

class InfoBase:
    def __init__(self, rawdata):
        self._rawdata = rawdata

class Request(InfoBase):
    def __init__(self, rawdata):
        super(Request, self).__init__(rawdata)
        self._message = ''
        self._headers = {}
        self._body = None

    @property
    def message(self):
        if not self._message:
            self._message = self._rawdata.split(b'\r\n')[0]
        return self._message

    @property
    def headers(self):
        if not self._headers:
            self._headers = {
                h.split(b':')[0].strip(): h.split(b':', 1)[1].strip()
                for h in self._rawdata.split(b'\r\n\r\n')[0].split(b'\r\n')[1:]
            }
        return self._headers

    @property
    def body(self):
        if self._body is None:
            self._body = self._rawdata.split(b'\r\n\r\n', 1)[1]
        return self._body

class ClientRequest(Request):
    @property
    def method(self):
        return self.message.split(b' ')[0]

class ServerRequest(Request):
    @property
    def status(self):
        return self.message.split(b' ')[1]

    @property
    def content_type(self):
        ret = None
        try:
            ret = self.headers[b'Content-Type'].split(b';')[0].strip()
        except KeyError:
            pass
        return ret

class MetaData(InfoBase):
    def __init__(self, rawdata):
        super(MetaData, self).__init__(rawdata)
        self._timing = {}

    @property
    def timing(self):
        if not self._timing:
            dom = parseString(self._rawdata.decode('utf-8-sig'))
            item = dom.getElementsByTagName('SessionTimers')[0]

            for key in item.attributes.keys():
                self._timing[key] = item.attributes[key].value

        return self._timing

def main():
    import argparse
    parser = argparse.ArgumentParser(description='saz file parser')
    parser.add_argument('filename', help='saz file name')
    args = parser.parse_args()

    sazfile = SazFile(args.filename)
    static_count, server_time, dserver_time = 0, 0.0, 0.0
    download_time, ddownload_time = 0.0, 0.0
    dns_time, tcp_connec_time, https_handshake_time, gateway_time = 0, 0, 0, 0
    ddns_time, dtcp_connec_time, dhttps_handshake_time, dgateway_time = 0, 0, 0, 0
    for num, session in enumerate(sazfile.sessions):
        if session.is_static:
            static_count += 1
            server_time += session.server_time
            download_time += session.download_time
            dns_time += session.dns_time
            tcp_connec_time += session.tcp_connec_time
            https_handshake_time += session.https_handshake_time
            gateway_time += session.gateway_time
        else:
            dserver_time += session.server_time
            ddownload_time += session.download_time
            ddns_time += session.dns_time
            dtcp_connec_time += session.tcp_connec_time
            dhttps_handshake_time += session.https_handshake_time
            dgateway_time += session.gateway_time
    print(
        "{:8},{:8},{:8},{:8},{:8},{:8},{:8},{:8},{:8},{:8},{:8},{:8},{:8},{:8}".format(
            sazfile.session_num,
            static_count,
            round(server_time, 2),
            round(dserver_time, 2),
            round(download_time, 2),
            round(ddownload_time, 2),
            dns_time, ddns_time,
            tcp_connec_time, dtcp_connec_time,
            https_handshake_time, dhttps_handshake_time,
            gateway_time, dgateway_time,
        )
    )

if __name__ == '__main__':
    main()


import json
import gzip
import pandas as pd
import re

import exclude_functions

class CSVBuilder:
    def __init__(self, config, file_manager):
        self.config = config
        self.file_manager = file_manager

        # Load the column functions.
        self.column_functions = self.load_column_functions()

    def load_column_functions(self):
        column_functions = {}

        for column, func_name in self.config.get('exclude_funcs', {}).items():
            # Use getattr to get the function from the module.
            func = getattr(exclude_functions, func_name, None)

            # If the function exists, add it to the column functions.
            if func is not None:
                column_functions[column] = func

        return column_functions

    def lines(self, s, splitter='\n', regex=True):
        if not regex:
            splitter = re.escape(splitter)

        start = 0

        for m in re.finditer(splitter, s):
            begin, end = m.span()
            if begin != start:
                yield s[start:begin]
            start = end

        if s[start:]:
            yield s[start:]

    def line_to_dict(self, line):
        # Convert the line into a dictionary.
        data = json.loads(line)

        # Check for column exclusions.
        for column, func in self.column_functions.items():
            if column in data and func(data[column]):
                return None

        return data

    def process_gz_to_csv(self, gz_path, csv_path):
        # Process each gz file to csv.
        with gzip.open(gz_path, 'rb') as f:
            file_content = f.read().decode()
            useful_data = []

            for line in self.lines(file_content, regex=False):
                if line:
                    try:
                        line_dict = self.line_to_dict(line)
                        if line_dict is not None:
                            useful_data.append(line_dict)
                    except Exception as e:
                        print(f"Error processing line: {e}")
                        self.file_manager.error_lines += 1

            # Convert the data into a DataFrame.
            df = pd.DataFrame(useful_data)

            # Save the DataFrame to a CSV file.
            df.to_csv(csv_path, index=None)

    def process_files(self):
        # Get the list of gz files.
        gz_files = self.file_manager.get_gz_chunk_files()

        # Process each gz file.
        for gz_file in gz_files:
            csv_path = self.file_manager.get_csv_path(gz_file)
            self.process_gz_to_csv(gz_file, csv_path)

        # Merge the csv files into one.
        self.file_manager.merge_csv_files()

        # Cleanup the temporary files.
        self.file_manager.cleanup()


from ebooklib import epub

from src.chapters import ChapterRequester

class TextConverter:
    def __init__(self, title, begin, end):

        self.title  = title
        self.begin  = begin
        self.end  = end

        self.book = epub.EpubBook()
        self.initialize_book()
        self.save_book()

    def initialize_book(self):
        self.book.set_identifier('id{begin}{end}'.format(begin = self.begin, end=self.end))
        self.book.set_title(self.title)
        self.book.set_language('en')

    def save_book(self):
        self.chapters_list = list(range(self.begin, self.end + 1))
        print("Compiling book")
        for chapt in self.chapters_list:

            c = ChapterRequester(title=self.title, number=chapt)

            c1 = epub.EpubHtml(title=str(chapt), file_name='chap_{id}.xhtml'.format(id=chapt), lang='hr')
            c1.content=u'
{heading}

{text}
'.format( heading=c.chapter.title, text=c.chapter.text_content)
            self.book.add_item(c1)
            self.book.spine.extend((c1,))
            self.book.toc.extend((c1,))

        style = 'BODY {color: white;}'
        nav_css = epub.EpubItem(uid="style_nav", file_name="style/nav.css", media_type="text/css", content=style)

        self.book.add_item(nav_css)

        self.book.add_item(epub.EpubNcx())
        self.book.add_item(epub.EpubNav())

        epub.write_epub('{title}.epub'.format(title = self.title), self.book, {})

    def convert_epub(self):
        self.book.set_identifier('id123456')
        self.book.set_title(self.title)
        self.book.set_language('en')
        #self.book.add_author(self.author)

        self.c1 = epub.EpubHtml(title=self.title, file_name='chap_01.xhtml', lang='hr')
        self.c1.content=u'
Intro heading

{text}
'.format(text=self.text)
        self.book.add_item(self.c1)

        self.book.toc = (epub.Link('chap_01.xhtml', 'Introduction', 'intro'),
             (epub.Section('Simple book'),
             (self.c1, ))
            )

        self.book.add_item(epub.EpubNcx())
        self.book.add_item(epub.EpubNav())

        style = 'BODY {color: white;}'
        nav_css = epub.EpubItem(uid="style_nav", file_name="style/nav.css", media_type="text/css", content=style)

        self.book.add_item(nav_css)

        self.book.spine = ['nav', self.c1]

        epub.write_epub('test.epub', self.book, {})

    def multithreaded_save(self):
        print('Processing with threads...')

        self.num_list = list(range(self.number - self.go_back,self.number))

        def read_data(num):
            c = ChapterRequester(
            number=num, site=self.site, title=self.title)

            self.collected_text.append(c.chapter.text_content)

        threadpool = Threadpool(processes = self.NUM_THREADS)
        self.collected_text = []

        results = threadpool.map(read_data, self.num_list)

        #print(self.collected_text)

        self.collected_text = " ".join(self.collected_text)

        return self


# goals: scatter plots of the blood laboratory tests next to each other. With the color showing positive and negative test results.

# Import modules
import pandas as pd
from bokeh.models import ColumnDataSource, Jitter, Legend, LegendItem
from bokeh.layouts import column, gridplot
from bokeh.models.annotations import Legend
from bokeh.plotting import figure, output_file, show
from bokeh.transform import jitter

pd.set_option('display.max_columns', None)

# import the dataset
df = pd.read_excel("dataset.xlsx")

# select the data that is going te be plotted
SELECTION = [
    'SARS-Cov-2 exam result',
    'Patient age quantile',
    'Hematocrit',
    'Hemoglobin',
    'Platelets',
    'Red blood Cells',
    'Lymphocytes',
    'Mean corpuscular hemoglobin concentration (MCHC)',
    'Leukocytes',
    'Basophils',
    'Mean corpuscular hemoglobin (MCH)',
    'Eosinophils',
    'Mean corpuscular volume (MCV)',
    'Monocytes',
    'Red blood cell distribution width (RDW)',
    #'Serum Glucose', <-- averaged around 4 values per quantile, which is not enough
    ]

colorPositive = "blue"      # color of the positive markers
colorNegative = "red"       # color of the negative markers

# title of the plot
TITLE = "Several blood chemicals versus Age quantile"

''' Data cleaning =============================================================='''
# selecting the required data
df_blood = df[SELECTION].copy()
print(df_blood)

# seperating dataset by negative and positive test results
dfPositive = df_blood[df_blood['SARS-Cov-2 exam result'] == "positive"]
dfNegative = df_blood[df_blood['SARS-Cov-2 exam result'] == "negative"]

# seperating age quantiles of negative and positive patients
dfPosAge = dfPositive['Patient age quantile']
dfNegAge = dfNegative['Patient age quantile']

# removing datasets that are no longer needed
del dfPositive['SARS-Cov-2 exam result']
del dfNegative['SARS-Cov-2 exam result']

print(dfPositive)

# set output file
output_file("test.html", title = "Scatter visualization of blood tests")

# convert dataframes to dictionaries
dcPositive = dfPositive.to_dict("list")
dcNegative = dfNegative.to_dict("list")

#print(dcBlood['Patient age quantile'])

list = list(dcPositive)                         # create list with all columns names of the dataset (which also includes age)
list.remove('Patient age quantile')             # remove ages from list, as they are already seperated into their own variable

figures = []                                    # list of figures to be made

# convert dataframes to datasources for the figures
sourcePos = ColumnDataSource(dfPositive)
sourceNeg = ColumnDataSource(dfNegative)

for index in list:                                  # for every blood column in the dataset...

    # lists for the Legends of the plot
    posLeg = []
    negLeg = []

    scatter = figure(                                   # make a scatterplot with the following properties...
        title = index,                                  # sets the titel of the plot to the name of the column
        plot_width=500,                                 # plot width in pixels
        plot_height=300,                                # plot height in pixels
        tools = "save, pan, reset, wheel_zoom",         # interaction tools with the plot
        x_axis_label='age quantile',                    # x-axis data from the source
        y_axis_label='standardized test result'         # y-axis data from the source
    )

    # paint data in the 'index' column as scattered circles
    p = scatter.circle(                                 # ... make point for the scatterplots with...
        x = jitter("Patient age quantile", 0.5),        # ... 'Patient age quantile' as x-coordinate
        y = index,                                      # ... 'index' (current blood chemical) as y-coordinate
        size = 4,                                       # size of the circles in pixels
        color = colorPositive,                          # color of the dots
        alpha = 0.5,                                    # opacity of the dots
        source = sourcePos,                             # datasource for the dots (positive patients only)
        muted_alpha = 0.1                               # opacity of dots that are disabled by the user when clicking on the legend item
    )
    n = scatter.circle(                                 # ... make point for the scatterplots with...
        x=jitter("Patient age quantile", 0.5),          # ... 'Patient age quantile' as x-coordinate
        y= index,                                       # ... 'index' (current blood chemical) as y-coordinate
        size=4,                                         # size of the circles in pixels
        color=colorNegative,                            # color of the dots
        alpha=0.5,                                      # opacity of the dots
        source = sourceNeg,                             # datasource for the dots (negative patients only)
        muted_alpha=0.1                                 # opacity of dots that are disabled by the user when clicking on the legend item
    )

    # add the painted dots to the Legend
    posLeg.append(("Covid-19 positive", [p]))
    negLeg.append(("Covid-19 negative", [n]))

    # construct the legend
    legend = Legend(items=posLeg + negLeg)              # create Legend object
    legend.click_policy = "mute"                        # specify what happens when the user click on a legend item
    scatter.add_layout(legend, 'right')                 # position of the Legend relative to the corresponding plot

    figures.append(scatter)                             # ... add the scatterplot to the list of figures

# create grid of plots using a list comprehension, this creates a list of lists where each inner list contains 3 consequtive elements from 'figures',
# !! this only works if the length of 'figures' is exactly divisable by 3.  !!
splom = gridplot([ [figures[i], figures[i+1], figures[i+2] ] for i in range(0, len(figures)-1, 3) ])

show(splom)  # show the plots on the page


import json
import os
import pickle
import re
import string

def tokenize(s):
    re_tok = re.compile('([{}“”¨«»®´·º½¾¿¡§£₤‘’])'.format(string.punctuation))
    return re_tok.sub(r' \1 ', s).split()

def load_models():
    models = {}
    for file in os.listdir('models'):
        with open(os.path.join('models', file), 'rb') as m, \
                open(os.path.join('ratio', file.replace('model', 'r')), 'rb') as r:
            models[file] = {'model': pickle.load(m), 'ratio': pickle.load(r)}
    return models

def evaluate_toxicity(body, vec):

    text = vec.transform([body])
    pred_sum = 0
    for one_model in models:
        model = models[one_model]['model']
        ratio = models[one_model]['ratio']
        predict = model.predict_proba(text.multiply(ratio))[:, 1]
        pred_sum += float(predict[0])
    return pred_sum

def evaluate_posts():
    file_dir = os.listdir('comments2018')
    num = 1
    index = 1
    with open('data/vectoriser.pkl', 'rb') as vec:
        vec = pickle.load(vec)
    so_results = []
    for file in file_dir:
        with open(os.path.join('comments2018', file), 'r') as f:
            so_dict = json.load(f)
            for post in so_dict:
                post['Toxicity'] = evaluate_toxicity(post['Body'], vec)
                if post['Toxicity'] > 0.5:
                    so_results.append(post)
                    num += 1
                if num % 1000 == 0:
                    with open('results/so_results_com_2018_{}.json'.format(index), 'w') as f:
                        json.dump(so_results, f)
                        so_results = []
                        print('file {}'.format(index))
                        index += 1

    with open('results/so_results_com_2018_{}.json'.format(index+1), 'w') as f:
        json.dump(so_results, f)

if __name__ == "__main__":
    print('start')
    models = load_models()
    print('load models')
    evaluate_posts()


#!/usr/bin/env python3

import os
from fpdf import FPDF
from subprocess import Popen, PIPE
from PIL import Image

title = "ISSessions CTF 2021 - Documentation Booklet"

class PDF(FPDF):
    def header(self):
        pdf.set_title(title)
        pdf.set_author("ISSessions")
        img = Image.new("RGB", (215, 300), "#0D0D0D")
        img.save("background.png")
        pdf.image("background.png", x=0, y=0, w=215, h=300, type="", link="")

        pdf.set_text_color(148, 152, 166)
        pdf.set_draw_color(42, 45, 64)
        pdf.set_line_width(1.5)

        self.set_font("helvetica", "B", 18)
        w = self.get_string_width(title) + 6
        self.set_x((210 - w) / 2)
        self.cell(
            w,
            9,
            title,
            1,
            1,
            "C",
        )
        self.ln(10)

    def footer(self):
        self.set_y(-15)
        self.set_font("helvetica", "I", 8)
        self.set_text_color(128)
        self.cell(0, 10, "Page " + str(self.page_no()), 0, 0, "C")

    def challenge_title(self, category_name, challenge_name):
        self.set_text_color(13, 13, 13)
        self.set_font("helvetica", "B", 16)
        self.set_fill_color(88, 95, 115)
        self.cell(0, 8, category_name + ": " + challenge_name, 0, 1, "L", 1)
        self.ln(5)

    def challenge_body(self, name):
        with open(name, "rb") as fh:
            txt = fh.read().decode("latin-1")
        txt = txt.replace("\t", "    ")
        self.set_font("helvetica", "BU", 14)
        self.cell(0, 6, name.split("/").pop().replace(".txt", "").capitalize() + ":")
        self.ln(7)
        self.set_font("helvetica", "", 12)
        pdf.set_text_color(148, 152, 166)
        self.multi_cell(0, 5, txt)
        self.ln(4)

    def challenge_details(self, author_name, flag):
        pdf.set_text_color(242, 5, 5)
        self.set_font("helvetica", "B", 12)
        self.cell(17, 0, "Author: ")
        self.set_font("helvetica", "", 12)
        pdf.set_text_color(148, 152, 166)
        self.cell(50, 0, author_name)
        self.ln(5)
        self.set_font("helvetica", "B", 12)
        pdf.set_text_color(242, 5, 5)
        self.multi_cell(13, 0, "Flag:")
        self.set_font("helvetica", "", 12)
        pdf.set_text_color(148, 152, 166)
        if len(flag) > 1:
            self.ln(4)
            for i in flag:
                self.multi_cell(0, 0, " " * 12 + i)
                self.ln(4)
        else:
            self.multi_cell(0, 0, flag[0])

        self.ln(8)

    def write_challenge(self, category_name, challenge_name, file_names):
        grep = Popen(
            ["grep", "author:", root.replace("documentation", "challenge.yml")],
            stdout=PIPE,
        )
        author_name = str(grep.communicate()[0]).split(": ")[1][:-3]
        grep = Popen(
            [
                "egrep",
                r"\- (\")?monkeyCTF{.+}",
                root.replace(
                    "documentation",
                    "challenge.yml",
                ),
            ],
            stdout=PIPE,
        )
        flag = str(grep.communicate()[0])[4:-3]
        flag = flag.strip('"')
        flag = flag.split(r"\n- ")
        self.add_page(format="A4")
        self.challenge_title(category_name, challenge_name)
        self.challenge_details(author_name, flag)
        pdf.set_text_color(242, 220, 107)
        self.challenge_body(root + "/" + file_names[0])

pdf = PDF(format="A4")
pdf.set_margins(left=20, right=20, top=15)

for (root, dirs, files) in os.walk("../"):
    if (
        not root.startswith("./.")
        and root != "."
        and any("solution.txt" in i for i in files)
    ):
        category = root.replace("./", "")
        if category.endswith("/documentation"):
            challenge_name = category.split("/")[1]
            category_name = category.split("/")[0]
            pdf.write_challenge(category_name, challenge_name, ["solution.txt"])
            print(category_name + ": " + challenge_name)

pdf.output("ISSessionsCTF2022_Hosted_Documentation_Booklet.pdf", "F")


import torch
import torch.nn as nn
import torch.optim as optim

from zonotope import Zonotope
from networks import Normalization

from znet import zNet, zMaxSumOfViolations, zSumOfMaxIndividualViolations, zMaxViolation

class Analyzer:
    """
    Analyzer expected by `verifier.py`, to be run using `Analyzer.analyze()`.
    In terms of the attributes, the query to be answered is:
        ?"forall x in input_zonotope, net(x) labels x as true_label"?
    `self.zloss(self.znet(self.input_zonotope))` is a parameterized loss function with parameters `self.znet.lambdas` and `self.zloss.logit_lambdas`.
        If it returns a value <= 0, then the query is true.
    `analyze()` optimizes these parameters to minimize the loss.

    Attributes:
        znet (zNet): the network with zonotope variables and parameters lambda, s.t self.__net is self.znet "in the concrete"
        zloss (znet.zLoss): the loss function, with zonotope input, that translates the logical property to analyze
        input_zonotope (Zonotope): the zonotope to analyze (derived from inp and eps in the __init__)
        true_label (int): the true label of the input point
        __net (networks.FullyConnected || networks.Conv): the network to be analyzed (first layer: Normalization). kept for convenience
        __inp (torch.Tensor): a copy of the input point inp. kept for convenience
        __eps (float): a copy of the queried eps. kept for convenience

    Args:
        net: see Attributes
        inp (torch.Tensor): input point around which to analyze, of shape torch.Size([1, 1, 28, 28])
        eps (float): epsilon, > 0, eps.shape = inp.shape
        true_label (int): see Attributes
        nb_classes (int, optional): the number of classes to analyze, default=10. Allows to do testing without breaking the code.
    """

    def __init__(self, net, inp, eps, true_label, nb_classes=10):
        self.__net = net
        for p in net.parameters():
            p.requires_grad = False  # freeze the concrete network layers, to avoid doing useless computations
        self.__inp = inp
        self.__eps = eps
        self.true_label = true_label

        self.znet = zNet(net, input_shape=inp.shape[1:], nb_classes=nb_classes)
        # self.zloss = zMaxSumOfViolations(true_label=true_label, nb_classes=nb_classes)
        # self.zloss = zSumOfMaxIndividualViolations(true_label=true_label, nb_classes=nb_classes)
        self.zloss = zMaxViolation(true_label=true_label, nb_classes=nb_classes)

        upper = inp + eps
        lower = inp - eps
        upper.clamp_(max=1)  # clip input_zonotope to the input space
        lower.clamp_(min=0)
        a0 = (upper + lower) / 2  # center of the zonotope
        # A must have shape (nb_error_terms, *[shape of input])
        # for the input layer, there is 1 error term for each pixel, so nb_error_terms = inp.numel()
        A = torch.zeros(inp.numel(), *inp.shape[1:])  # torch.zeros(784, 1, 28, 28)
        mask = torch.ones(*inp.shape[1:], dtype=torch.bool)  # torch.ones(1, 28, 28, dtype=torch.bool)
        A[:, mask] = torch.diag(((upper - lower) / 2).reshape(-1))
        self.input_zonotope = Zonotope(A, a0)

    def analyze(self, verbose=False):
        """Returns True iff the `self.__net` is verifiably robust on `self.input_zonotope`
        Doesn't return until it is proved, i.e never returns False

        What the current implementation does:
            Run an optimizer on `self.znet.lambdas` and `self.zloss.logit_lambdas` to minimize `self.zloss(self.znet(self.input_zonotope))`.
            Return True when we find values of the lambdas s.t loss == 0
        """
        if verbose: print("entering Analyzer.analyze() with znet: \n{}".format(self.znet))

        if self.zloss.has_lambdas:
            optimizer = optim.Adam([self.zloss.logit_lambdas, *self.znet.lambdas], lr=0.043597391352031695)
            # zm.zReLU has the feature that setting the requires_gradient to False makes us use DeepZ, e.g:
            # self.zloss.logit_lambdas.requires_grad = False
        else:
            optimizer = optim.Adam([*self.znet.lambdas], lr=0.043597391352031695)

        inp_zono = self.input_zonotope
        if verbose:
            print("Analyzer.analyze(): performing the optimization on inp_zono: {}".format(inp_zono))
            print("Analyzer.analyze(): using loss function {} and optimizer {}".format(type(self.zloss), optimizer))
        # just run this optimizer ad infinitum.
        while_counter = 0
        while True:
            # print("optimizer parameters", optimizer.__getstate__()['param_groups'][0]['params'])  # useful for debugging
            if verbose:
                print("Analyzer.analyze(): iteration #{}".format(while_counter))
            optimizer.zero_grad()
            out_zono = self.znet(inp_zono, verbose=verbose)

            loss = self.zloss(out_zono, verbose=verbose)

            if loss <= 0:
                if verbose: print("Analyzer.analyze(): found loss<=0 (loss={}) after {} iterations. The property is proved.".format(loss.item(), while_counter))
                return True
            if verbose:
                print("Analyzer.analyze(): current loss:", loss.item())
                print("Analyzer.analyze(): doing loss.backward() and optimizer.step()")
            loss.backward()
            optimizer.step()

            with torch.no_grad(): # we can safely ignore grads here. They will be recomputed from scratch at the next evaluation of the loss anyway.
                for lambda_layer in self.znet.lambdas:
                    lambda_layer.clamp_(min=0, max=1)
                if self.zloss.has_lambdas:
                    self.zloss.logit_lambdas.clamp_(min=0, max=1)

            while_counter += 1

            """
            # for DEBUG: end the analysis early
            # let run for break_at_iter steps
            break_at_iter = 20
            if while_counter < break_at_iter:
                continue
            print()
            print("out_zono.A:\n{}\nout_zono.a0:\n{}".format(out_zono.A, out_zono.a0)) # since we're exiting, we can afford to print the results without cluttering the stdout
            # cannot use optimizer.state_dict bc it hides information (returns index of param instead of param tensor). Had to look into pytorch.optimize source code to find this.
            print("optimizer parameters:\n", optimizer.__getstate__()['param_groups'][0]['params'])
            # print("self.zloss.logit_lambdas:\n{}\nself.znet.lambdas:\n{}".format(self.zloss.logit_lambdas, self.znet.lambdas)) # should contain the same thing as what we just printed
            print("For convenience in testing, we exit now (after {} iterations), even though we still have time before timeout.".format(break_at_iter))
            return False
            """

        import warnings
        warnings.warn("Analyzer.analyze() is returning False, i.e. we're giving up even though we haven't timed out.")
        return False

    # Debugging utilities
    # ~~~~~~~~~~~~~~~~~~~

    def check_parameters(self):
        """A debugging utility.
        Check that the self.znet.lambdas is what we want (i.e the set of all the lambdas used as parameters).
        This also checks that self.znet only has the lambdas as active parameters (i.e all others have required_grad=False).
        """
        import zmodules as zm
        for zlayer in self.znet.zlayers:
            print(zlayer)
            for p in zlayer.parameters():
                # with torch.no_grad(): # to see that self.znet.lambdas does indeed reference the same thing (the next for loop will print all 1's)
                #     p.fill_(1)
                if p.requires_grad == True:
                    assert isinstance(zlayer, zm.zReLU)
                    print(p)
                else:
                    print("some frozen parameter (with requires_grad = False) of shape {}".format(p.shape))
        for lam in self.znet.lambdas:
            print(lam)

    def run_in_parallel(self, inp_zono=None, concrete_net=None):
        """A debugging utility.
        Runs a concrete network and the corresponding zNetwork in parallel to make some checks.
        Note that the concrete point and the zonotope center are NOT theoretically supposed to be identical, since we recenter the zonotope at each ReLU.
        """
        if inp_zono is None:
            inp_zono = self.input_zonotope
        if concrete_net is None:
            net = self.__net
            znet = self.znet
        else:
            net = concrete_net
            znet = zNet(concrete_net)
        next_point = inp_zono.a0
        next_zono = inp_zono
        assert torch.allclose(next_point, next_zono.a0)  # at the input layer the concrete point and the zono-center are identical
        for i in range(len(net.layers)):
            layer = net.layers[i]
            zlayer = znet.zlayers[i]
            print(layer, zlayer)
            next_point = layer(next_point)
            next_zono = zlayer(next_zono)
            # print(next_point.shape, next_zono.a0.shape)
            print("next_point (ground truth):\n", next_point)
            print("next_zono.a0:\n", next_zono.a0)
            print("next_zono.A:\n", next_zono.A)

    def make_dot_loss(self, gv_filename):
        """Use https://github.com/szagoruyko/pytorchviz to visualize the computation graph of the loss.
        Writes the result to gv_filename in .gv and .gv.pdf formats.
        Returns the corresponding graphviz.Digraph object.
        """
        try:
            import torchviz
            inp_zono = Zonotope(torch.zeros_like(self.input_zonotope.Z))
            out_zono = self.znet(inp_zono)
            loss = self.zloss(out_zono)
            dot = torchviz.make_dot(loss)
            dot.render(gv_filename, view=False)
            return dot
        except ImportError as err:
            import warnings
            warnings.warn("torchviz is not installed in the execution environment, so cannot make_dot. Skipping")

    def make_dot_znet(self, gv_filename):
        """Visualize the computation graph of the zNet."""
        try:
            import torchviz
            inp_zono = Zonotope(torch.zeros_like(self.input_zonotope.Z))
            out_zono = self.znet(inp_zono)
            out_aggr = torch.cat([out_zono.A, out_zono.a0], dim=0)
            dot = torchviz.make_dot(out_aggr)
            dot.render(gv_filename, view=False)
            return dot
        except ImportError as err:
            import warnings
            warnings.warn("torchviz is not installed in the execution environment, so cannot make_dot. Skipping")

    def make_dot_concrete(self, gv_filename):
        """Visualize the computation graph of concrete network `self.__net`."""
        try:
            import torchviz
            inp = torch.zeros(784, 1, 28, 28)
            for p in self.__net.parameters():
                p.requires_grad = True  # temporarily set it back to true (required for the computation graph)
            out = self.__net(inp)
            for p in self.__net.parameters():
                p.requires_grad = False
            dot = torchviz.make_dot(out)
            dot.render(gv_filename, view=False)
            return dot
        except ImportError as err:
            import warnings
            warnings.warn("torchviz is not installed in the execution environment, so cannot make_dot. Skipping")