# -*- coding: utf-8 -*-
"""
Energy Calibration Analysis v3.1

Lukas Kostal, 14.11.2023, ICL
"""

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import scipy.optimize as so
import scipy.signal as ss
import scipy.interpolate as si
import scipy.odr as sr
import itertools as it
import datetime

# gaussian function for fitting
def gauss(x, A, mu, sigma):
    y = A * np.exp(-((x - mu) / sigma)**2 / 2)
    return y

# gaussian function for fitting with ODR
def odr_gauss(B, x):
    A, mu, sigma = B
    y = A * np.exp(-((x - mu) / sigma)**2 / 2)
    return y

# polynomial function for fitting
def poly(x, *coef):
    if type(x) == np.float64:
        y = 0
    else:
        y = np.zeros(len(x))
    for i in range(0, len(coef)):
        y += coef[i] * x**i

    return y

# polynomial function for fitting with odr
def odr_poly(coef, x):
    y = np.zeros(len(x))
    for i in range(0, len(coef)):
        y += coef[i] * x**i
    return y

# function to load data
def load_mes(mes_num, src):
    # load the data as a dataset
    df = pd.read_csv(f'Data/Mes{mes_num}_{src}.csv')

    # slice into arrays
    t = df.iloc[:, 0].to_numpy()
    U = df.iloc[:, 4].to_numpy()
    N = df.iloc[:, 1].to_numpy()
    n = df.iloc[:, 2].to_numpy()
    g = df.iloc[0, -1]

    # remove nans from shorter columns
    t = t[~np.isnan(t)]
    U = U[~np.isnan(U)]
    n = n[~np.isnan(n)]
    N = N[~np.isnan(N)]

    return t, U, n, N, g

# function to combine bins
def cmb_bin(n_arr, N_arr, cmb):
    cmb = int(cmb)
    n_arr = n_arr[(len(n_arr) % cmb):]
    N_arr = N_arr[(len(N_arr) % cmb):]

    N_arr = np.sum(N_arr.reshape(-1, cmb), axis=1)
    n_arr = np.mean(n_arr.reshape(-1, cmb), axis=1)

    return n_arr, N_arr

# function to get reduced chi squared
def calc_chi(o, sig, e, dof):
    with np.errstate(all='ignore'):
        chi_arr = (o - e)**2 / (sig)**2

    chi_arr = chi_arr[~np.isinf(chi_arr)]
    chi = np.sum(chi_arr) / (len(chi_arr) - dof)

    return chi

# dataset number to use for calibration
mes_num = 3

# minimum no of events for combining bins
N_min = 1000

# order of calibration polynomial
cal_ord = 3

# factor of sigmas at which reduced chi sqaured is calculated
f_sig = np.sqrt(2 * np.log(2))

# load reference peaks as dataframe
df = pd.read_csv('Load/reference.csv', comment='#')

# slice to array of sources and energies
src = df.iloc[:, 0].to_numpy()
E = df.iloc[:, 1].to_numpy()
Eerr = df.iloc[:, 2].to_numpy()

# sort the sources with the energies
i_srt = np.argsort(src)
src = src[i_srt]
E = E[i_srt]
Eerr = Eerr[i_srt]

# get array of unique reference sources so remove duplicates
src_ref, i_rpt, n_rpt = np.unique(src, return_index=True, return_counts=True)

# get the number of reference sources and reference peaks
n_ref = len(src_ref)
nn_ref = np.sum(n_rpt)

# array to append arrays of energies for reference peaks
E_ref = np.zeros(n_ref, dtype=object)
Eerr_ref = np.zeros(n_ref, dtype=object)

# loop over all reference sources write the peak energies to array
for i in range(0, n_ref):
    E_ref[i] = E[i_rpt[i]:i_rpt[i] + n_rpt[i]]
    Eerr_ref[i] = Eerr[i_rpt[i]:i_rpt[i] + n_rpt[i]]

# load the background measurement
t_bg, U_bg, n_bg, N_bg, g_bg = load_mes(mes_num, 'Bg')

# calculate scaled channel no by accounting for gain
h_bg = n_bg / -g_bg

# calcualte total background measurement time and rate with error
T_bg = np.amax(t_bg)
R_bg = N_bg / T_bg
Rerr_bg = np.sqrt(N_bg) / T_bg

# array of reference peak energies and error in keV
E_pks = np.zeros(nn_ref)
Eerr_pks = np.zeros(nn_ref)

# array of scaled channel no from peak fit and error
h_pks = np.zeros(nn_ref)
herr_pks = np.zeros(nn_ref)

# array of reduced chi squared values for fitted peaks
chi_pks = np.zeros(nn_ref)

# loop over all reference sources with s as counting parameter
s=0
for i in range(0, n_ref):

    # load the data
    t, U, n, N, g = load_mes(mes_num, src_ref[i])

    # calculate scaled channel no by accounting for gain
    h = n / -g

    # calculate total measurement time and scale background and error
    T = np.amax(t)
    N_bg = R_bg * T
    Nerr_bg = Rerr_bg * T

    # prepare for combining bins
    cmb = 1
    N_cmb = N
    h_cmb = h

    # whole max no of counts in bin is less than specified combine bins
    while np.amax(N_cmb) < N_min:
        h_cmb, N_cmb = cmb_bin(h, N, cmb)
        cmb += 1
    h_cmb, N_cmb = cmb_bin(h, N, cmb)
    h_bg_cmb, N_bg_cmb = cmb_bin(h_bg, N_bg, cmb)

    # interpolate background with cubic spline for resampling
    bg_cs = si.CubicSpline(h_bg_cmb, N_bg_cmb)

    # resample background from cubic spline
    N_bg_rsmp = bg_cs(h_cmb)

    # subtract the background rate and propagate error
    h = h_cmb
    N = N_cmb - N_bg_rsmp
    Nerr = np.sqrt(N_cmb + np.abs(N_bg_rsmp))

    # minimum scaled channel below which peaks are not detected
    h_min = 0.08 * np.amax(h)
    i_min = np.argmin(np.abs(h - h_min))

    # find peaks and get their indices
    pks, _ = ss.find_peaks(N[i_min:], width=4)
    pks += i_min

    # no of reference peaks for the source
    n_pks = len(E_ref[i])

    # if no of found peaks is less than no of reference peaks raise exception
    if len(pks) < n_pks:
        raise Exception('Error: insufficient number of peaks found')

    # sort the peaks by height and take the required no of peaks starting from heighest
    pks = pks[np.argsort(N[pks])[-n_pks:]]

    # manual correction for 2nd Na22 peak
    if src_ref[i] == 'Na22':
        pks[0] = np.argmin(np.abs(h - 620))

    # array of optimised parameters, error on parameters, slice indices for each peak
    popt_arr = np.zeros(n_pks, dtype=object)
    perr_arr = np.zeros(n_pks, dtype=object)
    idx_arr = np.zeros(n_pks, dtype=object)

    # loop over each peak to fit
    for j in range(0, n_pks):

        # specify bounds on fitting parameters ±0.2 within the detected peak
        bnds_min = (N[pks[j]] * 0.8, h[pks[j]] * 0.8, 0)
        bnds_max = (N[pks[j]] * 1.2, h[pks[j]] * 1.2, 100)

        # initial parameteers from detecting peaks
        ig = np.array([N[pks[j]], h[pks[j]], 10])

        # perform preliiminary gaussian fit over entire spectrum
        popt, pcov = so.curve_fit(gauss, h, N, p0=ig, bounds=(bnds_min, bnds_max))

        # find indices at specified factor of sigma below and above peak
        idx_min = np.argmin(np.abs(h - (popt[1] - f_sig * popt[2])))
        idx_max = np.argmin(np.abs(h - (popt[1] + f_sig * popt[2]))) +1

        # slice spectrum to given factor of sigma
        h_fit = h[idx_min:idx_max]
        N_fit = N[idx_min:idx_max]
        Nerr_fit = Nerr[idx_min:idx_max]

        # perform final gaussian fit over part of spectrum
        popt, pcov = so.curve_fit(gauss, h_fit, N_fit, p0=popt, sigma=Nerr_fit, absolute_sigma=True)

        # estimate error on fitting parameters from covariance matrix
        perr = np.sqrt(np.diag(pcov))

        # write potinised parameters, error on parameters and slicing indices to arrays
        popt_arr[j] = popt
        perr_arr[j] = perr
        idx_arr[j] = np.array([idx_min, idx_max])

        # calculate expected no of counts from gaussian fit
        N_exp = gauss(h_fit, *popt)

        # calcualte reduced chi sqaured value
        chi = calc_chi(N_fit, Nerr_fit, N_exp, 3)

        # write reference peak energy, scaled channel no from fit, reduced chi sqaured
        E_pks[s] = E_ref[i][j]
        h_pks[s] = popt[1]
        Eerr_pks[s] = Eerr_ref[i][j]
        herr_pks[s] = perr[1]
        chi_pks[s] = chi

        # increment counting parameter
        s +=1

    # title for plotting
    tit = f'{src_ref[i]}, $g=${g:.2f}, $cmb=${cmb}, $\chi^2_\\nu=${chi_pks[s-1]:.1f}'

    # loop over each peak to append reduced chi squared value to title
    for j in range(1, n_pks):
        tit += f', {chi_pks[s-j-1]:.1f}'

    # plot each curve fit without saving
    plt.title(tit, pad=40)
    plt.xlabel(r'scaled MCA channel $h = \frac{n}{-g}$')
    plt.ylabel(r'counts $N$ (unitless)')
    plt.rc('grid', linestyle=':', color='black', alpha=0.8)
    plt.grid()

    plt.errorbar(h, N, yerr=Nerr, c='blue', label='data')
    plt.plot(h[pks], N[pks], '.', ms=8, c='red', zorder=8, label='peak')
    plt.axvline(x=h_min, ls='--', c='red', zorder=9, label='cutoff')

    # loop over each peak to plot gaussian fit
    for j in range(0, n_pks):
        h_plt = h[idx_arr[j][0]:idx_arr[j][1]]
        if j == 0:
            plt.plot(h_plt, gauss(h_plt, *popt_arr[j]), c='orangered', zorder=10, label='fit')
        else:
            plt.plot(h_plt, gauss(h_plt, *popt_arr[j]), c='orangered', zorder=10)

    plt.legend(loc=(0.1, 1.05), ncol=4)

    plt.show()

# array of 0 initial guess coefficients to specify polynomial order
ig = np.zeros(cal_ord+1)

# perform perliminary fit to get initial parameters for ODR
popt, pcov = so.curve_fit(poly, h_pks, E_pks, p0=ig)

# perform orthogonal distance regression with error in both variables
pmod = sr.Model(odr_poly)
pdat = sr.RealData(h_pks, E_pks, sx=herr_pks, sy=Eerr_pks)
podr = sr.ODR(pdat, pmod, beta0=popt)
pout = podr.run()

# get optimised parameters and error from orthogonal distance regression
popt = np.array(pout.beta)
perr = np.array(pout.sd_beta)

# array to hold expected scaled channel no from fit
h_exp = np.zeros(len(h_pks))

# loop over each reference peak
for i in range(0, len(h_pks)):

    # polynomial function with fit parameters offset by reference peak energy
    poly_inv = lambda x : poly(x, *popt) - E_pks[i]

    # use Newton-Raphson method to solve for expected scaled channel no
    h_exp[i] = so.newton(poly_inv, x0=h_pks[i])

# no of combinations of n object of 2 distinct types (2^n)
n_combs = 2**len(popt)

# create array of all possible combinations of +, -
combs = list(it.product([1, -1], repeat=len(popt)))
combs = np.array(combs)

# 2D array to hold expected scalled channel no for all possible combinations of fit parameters
h_exp_combs = np.zeros((len(h_pks), n_combs))

# loop over all possible combinations of errors on fit parameters
for i in range(0, n_combs):
    popt_combs = popt + combs[i] * perr

    # loop over each reference peak
    for j in range(0, len(h_pks)):

        # polynomial function with fit parameters offset by reference peak energy
        poly_inv = lambda x : poly(x, *popt_combs) - E_pks[j]

        # use Newton-Raphson method to solve for expected scaled channel no
        root, info = so.newton(poly_inv, x0=h_pks[j], disp=False, full_output=True)

        if info.converged and root > 0:
            h_exp_combs[j][i] = root
        else:
            h_exp_combs[j][i] = np.nan

# take halve of maximum difference in combinations to be error in expected value
herr_exp = (np.nanmax(h_exp_combs, axis=1) - np.nanmin(h_exp_combs, axis=1)) / 2

# combine error in measured and expected scaled channel no in quadrature for chi calcualtion
herr_chi = np.sqrt(herr_pks**2 + herr_exp**2)

# calculate reduced chi sqaured for polynomial calibration
chi = calc_chi(h_pks, herr_chi, h_exp, len(popt))

# array of scaled channel no for plotting
h_plt = np.linspace(0, np.amax(h_pks), 100)

# 2D array to hold energies for all possible combinations of fit parameters
E_fit_combs = np.zeros((len(h_plt), n_combs))
E_res_combs = np.zeros((len(h_pks), n_combs))

# loop over all possible combinations of errors on fit parameters
# for each calcualte expected energy
for i in range(0, n_combs):
    popt_combs = popt + combs[i] * perr
    E_fit_combs[:, i] = poly(h_plt, *popt_combs)
    E_res_combs[:, i] = poly(h_pks, *popt_combs)

# take highest and lowest energies for plotting uncertainty region
E_fit_min = np.amin(E_fit_combs, axis=1)
E_fit_max = np.amax(E_fit_combs, axis=1)

# get residuals from polynomial calibration
E_res = E_pks - poly(h_pks, *popt)

# calculate error in residuals from error in fitting parameters
Eerr_res = np.sqrt(Eerr_pks**2 + (np.ptp(E_res_combs, axis=1) / 2)**2)

# get timestamp for the calibration
ts = datetime.datetime.now()
ts = ts.replace(microsecond=0)

# write calibration coefficients to .csv file
with open('Load/calibration.csv', 'w') as cal:
    print('order, coefficient, uncertainty', file=cal)

    for i in range(0, cal_ord+1):
        print(f'{i:.0f}, {popt[i]}, {perr[i]}', file=cal)

    print(f'# dataset: Mes{mes_num}', file=cal)
    print(f'# rcs value: {chi:2f}', file=cal)
    print(f'# timestamp: {ts}', file=cal)

# parameters for plotting calibration curve
plt.figure(1)
plt.title(f'Calibration Curve dataset: Mes{mes_num}, order: {cal_ord}, $\chi^2_\\nu=$ {chi:.2f}', pad=40)
plt.xlabel(r'scaled ADC channel $h = \frac{n}{-g}$')
plt.ylabel(r'energy $E_\gamma$ (keV)')
plt.rc('grid', linestyle=':', color='black', alpha=0.8)
plt.grid()

# plot the datapoints and uncertainty
plt.errorbar(h_pks, E_pks, xerr=herr_pks, yerr=Eerr_pks, fmt='.', c='blue', capsize=4, label='data')
plt.plot(h_plt, poly(h_plt, *popt), c='orangered', label='polynomial')
plt.fill_between(h_plt, E_fit_min, E_fit_max, color='royalblue', alpha=0.3, label='uncertainty')

plt.legend(loc=(0.1, 1.05), ncol=3)

# save the plot
plt.savefig(f'Output/Calibration_mes{mes_num}.png', dpi=300, bbox_inches='tight')
plt.show()

# parameters for plotting residuals
plt.figure(2)
plt.title(f'Calibration Residuals dataset: Mes{mes_num}, order: {cal_ord}, $\chi^2_\\nu=$ {chi:.2f}')
plt.xlabel(r'scaled ADC channel $h = \frac{n}{-g}$')
plt.ylabel(r'energy $E_\gamma$ (keV)')
plt.rc('grid', linestyle=':', color='black', alpha=0.8)
plt.grid()

# plot the datapoints and uncertainty
#plt.plot(h_pks, E_res, 'x', c='blue')
plt.errorbar(h_pks, E_res, xerr=herr_pks, yerr=Eerr_res, fmt='.', c='blue', capsize=4)

# save the plot
plt.savefig(f'Output/Residuals_mes{mes_num}.png', dpi=300, bbox_inches='tight')
plt.show()


import random
k = 2
contagem_estados = open("tmp/3.1.contagem-estados.txt", "r")
contagem_palavras = open("tmp/3.0.contagem-palavras.txt", "r")

modelo = {}

for contagem_palavra in contagem_palavras:
  estado,palavra,contagem = contagem_palavra.strip().split("\t")
  if estado not in modelo: modelo[estado] = {}
  modelo[estado][palavra] = int(contagem)

for contagem_estado in contagem_estados:
  estado,contagem = contagem_estado.strip().split("\t")
  contagem = int(contagem)
  step = 0.0
  for palavra in modelo[estado]:
    frequency = step + (modelo[estado][palavra] * 1.0/contagem)
    modelo[estado][palavra] = (step, frequency)
    step = frequency

def sorteia(estado):
  n = random.random()

  for palavra, v in estado.items():
    if n >= v[0] and n <= v[1]: return palavra

while(True):
  entrada = input("Escreva " + str(k) + " que produziremos um parágrafo com 20 palavras. \".\" para sair:\n")

  if entrada == ".":
    break

  palavras = ",".join(map(lambda x: x.strip(" .,;!?/\"\'+-)(:\n\t"), entrada.lower().split()))
  print(palavras)

  if(palavras in modelo):
    print("\nAutosugestão: ")

    print(sorteia(modelo[palavras]))
    print("\n")
  else:
    print("Sem sugestões.\n")


# buildifier: disable=module-docstring
# buildifier: disable=function-docstring
def exercise_the_api():
    var1 = apple_common.platform_type  # @unused
    var2 = apple_common.AppleDynamicFramework  # @unused

exercise_the_api()

# buildifier: disable=rule-impl-return
def my_rule_impl(ctx):
    _ignore = [ctx]  # @unused
    return struct()

# buildifier: disable=unsorted-dict-items
apple_related_rule = rule(
    implementation = my_rule_impl,
    doc = "This rule does apple-related things.",
    attrs = {
        "first": attr.label(mandatory = True, allow_single_file = True),
        "second": attr.string_dict(mandatory = True),
        "third": attr.output(mandatory = True),
        "fourth": attr.bool(default = False, mandatory = False),
    },
)


import requests
from collections import defaultdict

from django.conf import settings
from django.utils.translation import ugettext as _

from timetable.constants import LESSON_TYPES, WEEK_DAYS
from timetable.entities import APIEntity, Group, Teacher
from timetable.exceptions import StopExecution
from timetable.models import Chat

class KPIHubTimetable:
    """Class for interacting with rozklad.hub.kpi.ua as timetable backend"""
    def __init__(self, chat: Chat, entity: APIEntity, parameters: dict):
        self.chat = chat
        self.entity = entity
        self.parameters = parameters
        self.timetable = self._get_timetable()

    def execute(self, command):
        """Execute command using previously retrieved timetable.

        :param command: passed command without parameters
        :type command: str
        """
        # If API returned empty array
        if not self.timetable:
            if command == "/next":
                self.timetable = self._get_next_lesson()
            else:
                self._send(_("Пар нет, наслаждайся."))
                return

        # Transform timetable dictionary to readable form
        result = defaultdict(lambda: defaultdict(list))
        for lesson in self.timetable:
            result[lesson['week']][lesson['day']].append(
                self._format_lesson(lesson))

        # Send prepared messages
        for week_number, week_timetable in result.items():
            for day, lessons_list in week_timetable.items():
                header = "{} ({} {}):\n".format(WEEK_DAYS[day], week_number,
                                                _("неделя"))
                self._send(header + '\n'.join(sorted(lessons_list)))

    def _format_lesson(self, lesson: dict):
        """Format API lesson response to readable form"""
        lesson_type = " ({})".format(LESSON_TYPES[lesson['type']])\
            if lesson['type'] in LESSON_TYPES else ""

        rooms_list = ", ".join(lesson['rooms_full_names']) if lesson['rooms']\
            else _("расположение неизвестно")

        formatted_lesson = "*{}*: {}{} - {}".format(lesson['number'],
                                                    lesson['discipline_name'],
                                                    lesson_type,
                                                    rooms_list)
        # If "T" parameter has been passed, add teachers names to response.
        if hasattr(self.parameters, 'print_teacher'):
            if lesson['teachers']:
                formatted_lesson += "— {}\n".join(lesson['teachers_short_names'])
            else:
                formatted_lesson += "— {}\n".format(_("неизвестно"))
        return formatted_lesson

    def _get_next_lesson(self):
        """Custom function for handling not trivial cases for '/next' command"""
        # There are 3 possible scenarios:
        # 1. There is no next lesson, but there are lessons today in future.
        query_parameters = {
            'week': self.parameters.week,
            'day': self.parameters.day,
            'number__gt': self.parameters.number,
        }
        self.timetable = self._get_timetable(query_parameters)
        if not self.timetable:
            # 2. There are no lessons today, and we need to switch to
            #    next day (but it can be saturday, or sunday, so we
            #    need to consider it)
            # Check current week
            query_parameters = {
                'week': self.parameters.week,
                'day__gt': self.parameters.day,
            }
            self.timetable = self._get_timetable(query_parameters)
            if not self.timetable:
                # Check next week
                query_parameters = {
                    'week': 3 - self.parameters.week,
                }
                self.timetable = self._get_timetable(query_parameters)

        if self.timetable:
            self.timetable = self.timetable[:1]  # Get only first lesson
        else:
            # 3. Timetable for this group/teacher is empty.
            self._send(_("К сожалению, для Вас в базе нет расписания :("))
            raise StopExecution()

    def _get_timetable(self, query_parameters={}) -> list:
        if not query_parameters:
            possible_query_parameters = ['week', 'day', 'number']

            for parameter in possible_query_parameters:
                if hasattr(self.parameters, parameter):
                    query_parameters[parameter] = getattr(self.parameters, parameter)

        if isinstance(self.entity, Group):
            query_parameters['groups'] = self.entity.resource_id
        elif isinstance(self.entity, Teacher):
            query_parameters['teachers'] = self.entity.resource_id
        query_parameters['limit'] = 100

        response = requests.get(settings.TIMETABLE_URL + '/lessons', query_parameters)
        return response.json()['results']

    def _send(self, text):
        """Shortcut for sending text response to user"""
        settings.BOT.send_message(self.chat.id, text=text, parse_mode='Markdown')


from uaip import *
import urllib.request
import re
key="Python"
for i in range(0,10):
    key=urllib.request.quote(key)
    thispageurl="http://weixin.sogou.com/weixin?oq=&query="+key+"&type=2&page="+str(i+1)+"&ie=utf8"
    thispagedata=ua_ip(thispageurl)
    print(len(thispagedata))
    pat1='<div class="txt-box">.*?href="(.*?)"'
    rst1=re.compile(pat1,re.S).findall(thispagedata)
    if(len(rst1)==0):
        print("这一页没有爬成功")
        continue
    for j in range(0,len(rst1)):
        thisurl=rst1[j]
        pat2='amp;'
        thisurl=thisurl.replace(pat2,"")
        print(thisurl)
        thisdata=ua_ip(thisurl)
        print("这篇文章爬取成功，长度为：" +str(len(thisdata)))
        fh=open("D:\\我的教学\\Python\\韬云教育-腾讯-Python爬虫\\rst\\weixin\\"+str(i)+str(j)+".html","w",encoding="utf-8")
        fh.write(thisdata)
        fh.close()


import json
from chatterbot import ChatBot
from chatterbot.trainers import ListTrainer
from chatterbot.trainers import ChatterBotCorpusTrainer
import os
import openai
import config
import dynamodb
from User import User

openai.api_key = os.environ['OPENAI_API_KEY']

FINAL_MESSAGE = 'gotta go will talk later'

with open("contexts.json", "r") as f:
    contexts = json.loads(f.read())

# Create a new chatbot named Charlie
chatbot = ChatBot('Doppelganger', logic_adapters=[
    {
        'import_path': 'chatterbot.logic.BestMatch',
        'default_response': FINAL_MESSAGE,
        'maximum_similarity_threshold': 0.80
    },
])

trainer = ListTrainer(chatbot)
trainer2 = ChatterBotCorpusTrainer(chatbot)

trainer2.train(
    "chatterbot.corpus.english.greetings",
)

trainer.train([
    "Thanks",
    "👍🏾 👍🏾",
])

trainer.train([
    "Thank you",
    "👍🏾 👍🏾",
])

trainer.train([
    "to you too",
    "👍🏾 👍🏾",
])

trainer.train([
    "okay",
    "👍🏾 👍🏾",
])

trainer.train([
    "what will you be up to",
    "nothing much",
])

trainer.train([
    "we miss you guys",
    "👍🏾 👍🏾",
])

trainer.train([
    "is this a new number",
    "yeah! still have old number but got ths one too. save it",
])

trainer.train([
    "new number",
    "yeah! still have old number but got ths one too. save it",
])

trainer.train([
    "new apartment",
    "gotta go will talk later",
])

trainer.train([
    "will you be coming for the holidays",
    "no leave days at all...",
])

trainer.train([
    "will you be coming for the Christmas",
    "no leave days at all...",
])

trainer.train([
    "will you be coming for the xmas",
    "no leave days at all...",
])

trainer.train([
    "vist us soon",
    "👍🏾 👍🏾",
])

trainer.train([
    "hello",
    "hello",
])

trainer.train([
    "how have you been",
    "good good",
])

trainer.train([
    "can i please borrow",
    "dont have extra cash now things a tight",
])

trainer.train([
    "visa fee is",
    "dont have extra cash now things are tight",
])

trainer.train([
    "money will pay back",
    "dont have extra cash now things are tight",
])

trainer.train([
    "help with bill",
    "dont have extra cash now things are tight",
])

trainer.train([
    "help with 100 bucks",
    "dont have extra cash now things are tight",
])

trainer.train([
    "can i borrow",
    "dont have extra cash now things are tight",
])

trainer.train([
    "can i borrow money",
    "dont have extra cash now things are tight",
])

trainer.train([
    "do you have extra money",
    "dont have extra cash now things a tight",
])

trainer.train([
    "do you have extra cash",
    "dont have extra cash now things are tight",
])

trainer.train([
    "need help with computer",
    "have you tried to google it",
])

trainer.train([
    "need help with website",
    "have you tried to google it",
])
trainer.train([
    "are you a bot",
    "ha ha maybe maybe not ;)",
])
trainer.train([
    "are you a robot",
    "ha ha maybe maybe not ;)",
])
trainer.train([
    "are you an AI",
    "ha ha maybe maybe not ;)",
])

trainer.train([
    "say hi to everyone",
    "👍🏾 👍🏾",
])

def reply(message, active_user):
    """
    returns a response to the input from chatterbot if confidence is zero else response is returned from openai
    then updates user conversation  in dynamodb
    :param message:
    :type message: str
    :param active_user:
    :type active_user:User
    :return response:
    """
    bot_reply = chatbot.get_response(message)
    active_user_relation = active_user.Relation
    data = contexts[active_user_relation]
    persona = data["persona"]

    if bot_reply.confidence == 0:
        active_user.Conversation.append("{}: {}".format(active_user_relation, message))
        active_user.Conversation.append("{}: ".format(persona))
        prompt = "\n".join(active_user.Conversation)

        response = get_open_ai_response(active_user_relation, persona, prompt)
        active_user.Conversation[-2] = ("{}: {}".format(active_user_relation, message))
        active_user.Conversation[-1] = ("{}: {}".format(persona, response))
    else:
        response = bot_reply.text
        active_user.Conversation.append("{}: {}".format(active_user_relation, message))
        active_user.Conversation.append("{}: {}".format(persona, response))

    dynamodb.UpdateActiveUserConversation(active_user)

    return response

def get_open_ai_response(active_user_relation, persona, prompt):
    """
         generates text reply based on user conversation context
    :param active_user_relation:
    :type: str
    :param persona:
    :type: str
    :param prompt:
    :type: str
    :return openai reply:
    :rtype: str
    """
    response = openai.Completion.create(
        engine="text-davinci-002",
        prompt=prompt,
        temperature=0.3,
        max_tokens=240,
        top_p=1.0,
        frequency_penalty=0.5,
        presence_penalty=0.0,
        stop=[" {}:", " {}:".format(active_user_relation, persona)]
    )

    if 'choices' not in response:
        return FINAL_MESSAGE

    ai_reply = response["choices"][0]["text"].strip()

    if unsafe(ai_reply):
        return FINAL_MESSAGE

    return ai_reply

def unsafe(content_to_classify):
    """
    Checks and detects whether text may be sensitive or unsafe
    :param content_to_classify:
    :type content_to_classify: str
    :return: str
    """
    response = openai.Completion.create(
        engine="content-filter-alpha",
        prompt="<|endoftext|>" + content_to_classify + "\n--\nLabel:",
        temperature=0,
        max_tokens=1,
        top_p=0,
        logprobs=10
    )
    output_label = response["choices"][0]["text"]

    # This is the probability at which we evaluate that a "2" is likely real
    # vs. should be discarded as a false positive
    toxic_threshold = -0.355

    if output_label == "2":
        # If the model returns "2", return its confidence in 2 or other output-labels
        logprobs = response["choices"][0]["logprobs"]["top_logprobs"][0]

        # If the model is not sufficiently confident in "2",
        # choose the most probable of "0" or "1"
        # Guaranteed to have a confidence for 2 since this was the selected token.
        if logprobs["2"] < toxic_threshold:
            logprob_0 = logprobs.get("0", None)
            logprob_1 = logprobs.get("1", None)

            # If both "0" and "1" have probabilities, set the output label
            # to whichever is most probable
            if logprob_0 is not None and logprob_1 is not None:
                if logprob_0 >= logprob_1:
                    output_label = "0"
                else:
                    output_label = "1"
            # If only one of them is found, set output label to that one
            elif logprob_0 is not None:
                output_label = "0"
            elif logprob_1 is not None:
                output_label = "1"

            # If neither "0" or "1" are available, stick with "2"
            # by leaving output_label unchanged.

    # if the most probable token is none of "0", "1", or "2"
    # this should be set as unsafe
    if output_label not in ["0", "1", "2"]:
        output_label = "2"

    return output_label == "2"

"""
import ast
# The following loop will execute each time the user enters input
while True:
    try:
        phonesDict = ast.literal_eval(os.environ["PHONES"])
        active_user = dynamodb.GetActiveUser(phonesDict["tunga"])
        user_input = input()
        bot_response = reply(user_input, active_user)
        print(bot_response)
    # Press ctrl-c or ctrl-d on the keyboard to exit
    except (KeyboardInterrupt, EOFError, SystemExit):
        break
"""


from aws_cdk import aws_lambda
from aws_cdk import aws_apigateway
from aws_cdk import core as cdk

from aws_solutions_constructs import aws_apigateway_lambda

from aws_cdk.aws_lambda_python import PythonFunction

class ServerlessEventsStack(cdk.Stack):

    def __init__(self, scope: cdk.Construct, construct_id: str, **kwargs) -> None:
        super().__init__(scope, construct_id, **kwargs)

        pyf = PythonFunction(
            self,
            "ApiGatewayToLambdaLambdaFunction",
            entry="serverless_events/lambda/", # required
            index="test.py", # optional, defaults to 'index.py'
            runtime=aws_lambda.Runtime.PYTHON_3_8
        )

        aws_apigateway_lambda.ApiGatewayToLambda(
            self,
            'ApiGatewayToLambda',
            existing_lambda_obj=pyf,
            api_gateway_props=aws_apigateway.RestApiProps(
                default_method_options=aws_apigateway.MethodOptions(
                    authorization_type=aws_apigateway.AuthorizationType.NONE,
                )
            )
        )


import logging
import logging.config
import os
import json

def setup_logging(default_path=os.path.dirname(os.path.realpath(__file__)) + "/../loggerConfiguration.json",
                  default_level=logging.INFO):
    """
    Setup logging configuration

    :param default_path: path to the logger configuration
    :param default_level: defines the default logging level if configuration file is not found (default: logging.INFO)
    """
    path = default_path
    if os.path.exists(path):
        with open(path, 'rt') as f:
            config = json.load(f)
        logging.config.dictConfig(config)
    else:
        logging.basicConfig(level=default_level)

setup_logging()


import numpy as np
import matplotlib.pyplot as plt
from labvaja1.vaja01 import loadImage
from labvaja1.displayImage import displayImage
from labvaja7.vaja07 import spatialFiltering

## 1. NALOGA:
if __name__ == "__main__":
    I = loadImage("labvaja7\data\cameraman-256x256-08bit.raw", [256, 256], np.uint8)
    # filtriranje z jedrom
    K = 1 / 16 * np.array([[1, 2, 1], [2, 4, 2], [1, 2, 1]])
    kI = spatialFiltering("kernel", iImage=I, iFilter=K)
    displayImage(kI, "Filtrirana slika - filtriranje z jedrom")

    # statisticno filtriranje
    sI = spatialFiltering(
        "statistical", I, iFilter=np.zeros((3, 3)), iStatFunc=np.median
    )
    displayImage(sI, "Filtrirana slika - statisticno filtriranje")

    # morfolosko filtriranje
    SE = np.array(
        [
            [0, 0, 1, 0, 0],
            [0, 1, 1, 1, 0],
            [1, 1, 1, 1, 1],
            [0, 1, 1, 1, 0],
            [0, 0, 1, 0, 0],
        ]
    )
    mI = spatialFiltering("morphological", I, iFilter=SE, iMorphOp="erosion")
    displayImage(mI, "Filtrirana slika - morfolosko filtriranje")

## 2.NALOGA:
def weightedAverageFilter(iM, iN, iValue=2):

    a = np.linspace(-(iN - 1) / 2, (iM - 1) / 2, iN)
    b = iValue**(-np.abs(a))*(iN-1)
    oFilter = np.outer(b, b)

    return oFilter

weightedAverageFilter(3,3)
weightedAverageFilter(5,5)

## 3.NALOGA:
if __name__ == "__main__":

    K1 = np.array([[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]])
    kI1 = spatialFiltering("kernel", iImage=I, iFilter=K1)
    displayImage(kI1, "Sobelov operator za x smer")

    K2 = np.array([[-1, -2, -1], [0, 0, 0], [1, 2, 1]])
    kI2 = spatialFiltering("kernel", iImage=I, iFilter=K2)
    displayImage(kI2, "Sobelov operator za y smer")

    kIam = np.sqrt(kI1**2 + kI2**2)
    displayImage(kIam, "Amplitudni odziv")

    kIfaza = np.arctan(kI2/kI1)
    displayImage(kIfaza, "Fazni odziv")

## 4.NALOGA:
if __name__ == "__main__":
    I = loadImage("labvaja7\data\cameraman-256x256-08bit.raw", [256, 256], np.uint8)
    displayImage(I, "original")

    K = np.array([[0.01, 0.08, 0.01], [0.08, 0.64, 0.08], [0.01, 0.08, 0.01]])
    kI = spatialFiltering("kernel", iImage=I, iFilter=K)
    displayImage(kI, "zglajena z gaussom")

    m = I - kI
    displayImage(m, "maska")

    g = I + 2*m
    displayImage(g, "izostreno")

## 5.NALOGA:

## 6.NALOGA:


from clearsight_2 import utils

class State:
    def __init__(self, name, resolutionTable, **properties):
        self.properties = properties

        if not type(resolutionTable) == utils.Table:
            raise SyntaxError("invalid state construction: state \"resolutionTable\" must be a utils.Table")

        for condition in resolutionTable.getCol("condition"):
            if type(condition) != type(lambda: None):
                raise SyntaxError("invalid state construction: condition must be a function")

        for callback in resolutionTable.getCol("callback"):
            if type(callback) != type(lambda: None):
                raise SyntaxError("invalid state construction: callback must be a function")

        for resultant in resolutionTable.getCol("resultant"):
            if type(resultant) != State:
                raise SyntaxError("invalid state construction: resultant must be a State")

        self.resolutionTable = resolutionTable

        self.name = name

    def resolve(self):
        for row in self.resolutionTable.keys():
            if self.resolutionTable[row]["condition"](self):
                self.resolutionTable[row]["callback"](self)
                return self.resolutionTable[row]["resultant"]

        return self


from django import forms
from django.utils.translation import gettext_lazy as _

from .models import Picture

class PictureForm(forms.ModelForm):
    class Meta:
        model = Picture

        fields = ('market',
                  'content',
                  )
        labels = {
            'market': _('Supermercado'),
            'content': _('Imagem'),
        }
        help_texts = {
            'market': _('O supermercado onde você está'),
            'content': _('Faça o upload da sua imagem.'),
        }
        error_messages = {
            'content': {
            },
        }
        widgets = {
        }


import numpy as np
import pandas as pd
import cv2
import seaborn as sns

from matplotlib import pyplot as plt

## load two csv files
df = pd.read_csv('../input/tuning_labels.csv', header=None,
                 names=['img_id', 'labels'])
description = pd.read_csv('../input/class-descriptions.csv')

## make a dictionary file
d={}
for i,j in zip(description.label_code.values, description.description.values):
    d[i]=j
count = pd.DataFrame(df['labels'].str.split().apply(lambda x:len(x)))
sns.countplot(data=count,x='labels')
plt.title('number of labels')
temp = df[count['labels']>6]
list = ['../input/stage_1_test_images/{}.jpg'.format(img_id) for img_id in temp.img_id.values]
ax=plt.figure(figsize=(12,12))
for num,i in enumerate(temp['labels'].apply(lambda x: x.split()).values):
    plt.subplot(3,2,2*num+1)
    plt.axis('off')
    filename = list[num]
    img = cv2.imread(filename)
    plt.imshow(img)
    names = [d[j] for j in i]
    for n,i in enumerate(names):
        plt.text(1500,10+n*100,i,fontsize=14,horizontalalignment='right')
## create a function called "plot_images"
def plot_images(num_label,max=60):
    temp = df[count['labels']==num_label]
    print(len(temp))
    if len(temp) > 60:
        temp = temp[:60]
    list = ['../input/stage_1_test_images/{}.jpg'.format(img_id) for img_id in temp.img_id.values]
    ax=plt.figure(figsize=(12,60))
    for num,i in enumerate(temp['labels'].apply(lambda x: x.split()).values):
        filename = list[num]
        img = cv2.imread(filename)
        img = cv2.resize(img, dsize=(1024, 600), interpolation=cv2.INTER_CUBIC)
        plt.subplot(30,4,2*num+1)
        plt.axis('off')
        plt.imshow(img)
        names = [d[j] for j in i]
        for n,i in enumerate(names):
            plt.text(1500,10+n*100,i,fontsize=10,horizontalalignment='left')
plot_images(6)
plot_images(5)
plot_images(4)
plot_images(3)
plot_images(2)
plot_images(1)


from pydriller import Modification,ModificationType
from pydriller import Commit
from os import path
import src.mining.CodeFeaturesMiners.constants_and_configs as constants_and_configs

from src.mining.CodeFeaturesMiners.AbstractMiner import AbstractMiner

class FileLevelFeatureMiner(AbstractMiner):
    def __init__(self, feature_classes) -> None:
        super().__init__(feature_classes)

    def mine_commit(self, repo_full_name, changeFile: Modification, history, index):
        self.features = {}
        self._mine_labels(changeFile, history[index], repo_full_name)
        if 'basic' in self.feature_classes:
            self._mine_basic(changeFile)
        if 'code_analysis' in self.feature_classes:
            self._mine_code_analysis(changeFile)
        if 'bag_of_words' in self.feature_classes:
            self._mine_bag_of_words(changeFile)
        if 'history' in self.feature_classes and history != None and index != None:
            self._mine_history_based(changeFile, history, index)
        return self.features

    def _mine_labels(self, changeFile: Modification, commit:Commit, repo_full_name):
        path = changeFile.new_path if changeFile.new_path != None else changeFile.old_path
        self._add_feature('label_sha', commit.hash)
        self._add_feature('label_commit_date', commit.committer_date)
        self._add_feature('label_repo_full_name', repo_full_name)
        self._add_feature('label_filepath', path.lower())

    def _mine_basic(self, changeFile: Modification):
        filename = changeFile.filename.lower()
        _, file_extension = path.splitext(filename)
        file_extension = file_extension.lower()

        has_ecosystem_files = {k:False for k in constants_and_configs.code_extensions}
        for ecosystem in constants_and_configs.code_extensions:
            for extension in constants_and_configs.code_extensions[ecosystem]:
                if file_extension == extension:
                    has_ecosystem_files[ecosystem] = True
                    break

        for k,v in has_ecosystem_files.items():
            self._add_feature(f'has_{k}_code', v)

        is_add = changeFile.change_type == ModificationType.ADD
        self._add_feature('is_add', is_add)
        is_rename = changeFile.change_type == ModificationType.RENAME
        self._add_feature('is_rename', is_rename)
        is_delete = changeFile.change_type == ModificationType.DELETE
        self._add_feature('is_delete', is_delete)
        is_modify = changeFile.change_type == ModificationType.MODIFY
        self._add_feature('is_modify', is_modify)

        removed_lines_count = changeFile.removed if changeFile.removed != None else 0
        self._add_feature('removed_lines_count', removed_lines_count)
        added_lines_count = changeFile.added if changeFile.added != None else 0
        self._add_feature('added_lines_count', added_lines_count)
        changed_lines_count = removed_lines_count + added_lines_count
        self._add_feature('changed_lines_count', changed_lines_count)
        removed_lines_ratio = removed_lines_count/changed_lines_count if changed_lines_count>0 else float('nan')
        self._add_feature('removed_lines_ratio', removed_lines_ratio)
        added_lines_ratio = added_lines_count/changed_lines_count if changed_lines_count>0 else float('nan')
        self._add_feature('added_lines_ratio', added_lines_ratio)

        if 'added' in changeFile.diff_parsed and 'deleted' in changeFile.diff_parsed:
            added_lines_set = set([x[0] for x in changeFile.diff_parsed['added']])
            removed_lines_set = set([x[0] for x in changeFile.diff_parsed['deleted']])
            modified_lines_count = len(added_lines_set.intersection(removed_lines_set))
            self._add_feature('modified_lines_count', modified_lines_count)
            modified_lines_ratio = modified_lines_count / changed_lines_count if changed_lines_count>0 else float('nan')
            self._add_feature('modified_lines_ratio', modified_lines_ratio)

        file_size = float("nan")
        if changeFile.source_code != None:
            file_size  = len(changeFile.source_code)
        elif changeFile.source_code_before != None:
            file_size  = len(changeFile.source_code_before)
        self._add_feature('file_size', file_size)

    def _mine_code_analysis(self, changeFile: Modification):
        if not changeFile.language_supported:
            return

        changed_methods_count = len(changeFile.changed_methods)
        self._add_feature('changed_methods_count', changed_methods_count)
        total_methods_count = len(changeFile.methods)
        self._add_feature('total_methods_count', total_methods_count)
        file_complexity = changeFile.complexity if changeFile.complexity != None else 0
        self._add_feature('file_complexity', file_complexity)
        file_nloc = changeFile.nloc if changeFile.nloc != None else 0
        self._add_feature('file_nloc', file_nloc)
        file_token_count = changeFile.token_count if changeFile.token_count != None else 0
        self._add_feature('file_token_count', file_token_count)

        max_method_token_count = 0
        avg_method_token_count = 0
        max_method_complexity = 0
        avg_method_complexity = 0
        max_method_nloc = 0
        avg_method_nloc = 0
        max_method_parameter_count = 0
        avg_method_parameter_count = 0
        keywords_in_method_names = {k:0 for k in constants_and_configs.better_security_keywords}

        for method in changeFile.changed_methods:
            method_name_lower = method.long_name.lower()
            max_method_token_count = max(max_method_token_count, method.token_count)
            avg_method_token_count = avg_method_token_count + method.token_count

            max_method_complexity = max(max_method_complexity, method.complexity)
            avg_method_complexity = avg_method_complexity + method.complexity

            max_method_nloc = max(max_method_nloc, method.nloc)
            avg_method_nloc = avg_method_nloc + method.nloc

            max_method_parameter_count = max(max_method_parameter_count, len(method.parameters))
            avg_method_parameter_count = avg_method_parameter_count + len(method.parameters)

            for k in constants_and_configs.better_security_keywords:
                if k in method_name_lower:
                    keywords_in_method_names[k] +=1

        self._add_feature('max_method_token_count', max_method_token_count)
        self._add_feature('max_method_complexity', max_method_complexity)
        self._add_feature('max_method_nloc', max_method_nloc)
        self._add_feature('max_method_parameter_count', max_method_parameter_count)

        save_file_changed_method_count = changed_methods_count if changed_methods_count > 0 else 1
        self._add_feature('avg_method_token_count', avg_method_token_count/save_file_changed_method_count)
        self._add_feature('avg_method_complexity', avg_method_complexity/save_file_changed_method_count)
        self._add_feature('avg_method_nloc', avg_method_nloc/save_file_changed_method_count)
        self._add_feature('avg_method_parameter_count', avg_method_parameter_count/save_file_changed_method_count)

        for k, v in keywords_in_method_names.items():
            self._add_feature(f'methods_with_{k}_count', v)

    def _mine_bag_of_words(self,changeFile: Modification):
        filename = changeFile.filename.lower()
        path = changeFile.new_path if changeFile.new_path != None else changeFile.old_path
        path = path.lower()

        test_in_filename = 'test' in filename
        self._add_feature('test_in_filename', test_in_filename)
        test_in_path = 'test' in path
        self._add_feature('test_in_path', test_in_path)

        diff_scaped = self._escape_separators(changeFile.diff.lower())

        file_content = ''
        if changeFile.source_code != None:
            file_content  = self._escape_separators(changeFile.source_code.lower())
        elif changeFile.source_code_before != None:
            file_content  = self._escape_separators(changeFile.source_code_before.lower())

        for k in constants_and_configs.better_security_keywords:
            if k in diff_scaped:
                self._add_feature(f'{k}_in_patch', True)
            else:
                self._add_feature(f'{k}_in_patch', False)
            if k in file_content:
                self._add_feature(f'{k}_in_file_content', True)
            else:
                self._add_feature(f'{k}_in_file_content', False)

    def _mine_history_based(self, changeFile, history, index):
        safe_path = changeFile.new_path if changeFile.new_path != None else changeFile.old_path

        changes_to_file_in_prev_50_commits = 0
        is_file_recently_added = False
        for offset in range(-50, -1):
            i = index + offset
            if i < 0:
                continue
            if i >= len(history):
                continue

            for mod in history[i].modifications:
                if safe_path == mod.new_path or safe_path == mod.old_path:
                    changes_to_file_in_prev_50_commits += 1
                    if mod.change_type == ModificationType.ADD:
                        is_file_recently_added = True

        self._add_feature('changes_to_file_in_prev_50_commits', changes_to_file_in_prev_50_commits)
        self._add_feature('is_file_recently_added', is_file_recently_added)

        changes_to_file_in_next_50_commits = 0
        is_file_recently_removed = False
        for offset in range(1, 50):
            i = index + offset
            if i < 0:
                continue
            if i >= len(history):
                continue

            for mod in history[i].modifications:
                if safe_path == mod.new_path or safe_path == mod.old_path:
                    changes_to_file_in_next_50_commits += 1
                    if mod.change_type == ModificationType.DELETE:
                        is_file_recently_removed = True

        self._add_feature('changes_to_file_in_next_50_commits', changes_to_file_in_next_50_commits)
        self._add_feature('is_file_recently_removed', is_file_recently_removed)


cycle_num = 1
X = 1

global signal_strengths
signal_strengths = list()

crt = [['.'] * 40 for i in range(6)]

with open("./input.txt") as file :
    instructions = [instruction.strip().split(' ') for instruction in file.readlines()]

instruction_num = 0

def run_cycle() :
    global cycle_num
    global crt

    if (cycle_num - 20) % 40 == 0 :
        print(cycle_num)
        curr_signal_strength = cycle_num * X
        signal_strengths.append(curr_signal_strength)

    crt_row_num = (cycle_num - 1) // 40
    pixel_num = (cycle_num - 1) % 40

    if (X - 1 <= pixel_num and pixel_num <= X + 1) :
        crt[crt_row_num][pixel_num] = '#'

    print("Cycle_num: %d, Row_num: %d, Pixel_num: %d, Sprite_Position: %d, Pixel: %s" % (cycle_num, crt_row_num, pixel_num, X, crt[crt_row_num][pixel_num]))

    cycle_num += 1

def render_image(crt):
    for row in crt :
        for pixel in row :
            print(pixel, end='')
        print()

while(cycle_num <= 240) :
    if (instruction_num in range(len(instructions))) :
        instruction = instructions[instruction_num]
        op_code = instruction[0]

        if op_code == "addx" :
            value = int(instruction[1])
            run_cycle()
            run_cycle()
            X += value
        else :
            run_cycle()

        instruction_num += 1
    else :
        run_cycle()

print(signal_strengths)
print("Sum of Signal Strengths:", sum(signal_strengths))

print(crt)
render_image(crt)


import pandas as pd
from pandas.plotting import scatter_matrix
import seaborn as sns; sns.set(context = 'paper', style = 'whitegrid')
import matplotlib.pyplot as plt
import numpy as np
from itertools import chain
from pprint import pprint

#Part 1
A = pd.read_excel("A.xlsx")
B = pd.read_excel("B.xlsx")
C = pd.read_excel("C.xlsx")
D = pd.read_excel("D.xlsx")
dataset = {"A" : A, "B" : B, "C" : C, "D" : D}

# Part 2
for x in dataset:
    sets = dataset[x]
    if sets.isna().values.any(): #using pandas method chaining
        print(f'Missing values in dataframe {x}: True')
    else:
        print(f'Missing values in dataframe {x}: False')

for x in dataset:
    sets = dataset[x]
    columns = list(sets.loc[:, sets.columns!= "id"].isna().sum()) #storing a variale to make the code easier to read
    print(f'Number of missing values in each column in {x}: {columns}')

for x in dataset:
    sets = dataset[x]
    sum_of_NaN = sum(list(sets.loc[:, sets.columns!= "id"].isna().sum()))
    print(f'total number of missing values in {x}: {sum_of_NaN}')

for x in dataset:
    sets = dataset[x]
    indexing = pd.isna(sets).any(1).to_numpy().nonzero()[0]
    objects = list(sets['id'][indexing])
    print(f'ID of rows with missing values in {x}: {objects}')

#Part 3
index_verdi = []
for x in dataset:
    sets = dataset[x]
    index_verdi.append(pd.isna(sets).any(1).to_numpy().nonzero()[0])
index_verdi = list(chain(*index_verdi))

print(index_verdi)

for y in dataset:
    dataset[y] = dataset[y].drop(index_verdi)

#part 4
sns.pairplot(dataset['A'])
rows_columns = dataset['A'].shape
print(f'amount of rows and columns in the dataset is: {rows_columns}')

dataset['B'].hist(figsize= (10,10))
rows_columns = dataset['B'].shape
print(f'Amount of rows and columns in the dataset is: {rows_columns}')

fig, ax = plt.subplots(figsize = (10,8))
sns.violinplot(data = dataset['C'], palette = "Set3", ax = ax, linewidth = 1)
rows_columns = dataset['C'].shape
print(f'Amount of rows and columns in the dataset is: {rows_columns}')

sns.heatmap(dataset['D'].corr(), linewidth = .1, vmax = 1, vmin = -1)
rows_columns = dataset['D'].shape
print(f'Amount of rows and columns in the dataset is: {rows_columns}')


#https://leetcode.com/problems/count-number-of-nice-subarrays/
#Given an array of integers nums and an integer k. A continuous subarray is called nice if there are k odd numbers on it._x000D_
#_x000D_
#Return the number of nice sub-arrays._x000D_
#_x000D_
# _x000D_
#Example 1:_x000D_
#_x000D_
#_x000D_
#Input: nums = [1,1,2,1,1], k = 3_x000D_
#Output: 2_x000D_
#Explanation: The only sub-arrays with 3 odd numbers are [1,1,2,1] and [1,2,1,1]._x000D_
#_x000D_
#_x000D_
#Example 2:_x000D_
#_x000D_
#_x000D_
#Input: nums = [2,4,6], k = 1_x000D_
#Output: 0_x000D_
#Explanation: There is no odd numbers in the array._x000D_
#_x000D_
#_x000D_
#Example 3:_x000D_
#_x000D_
#_x000D_
#Input: nums = [2,2,2,1,2,2,1,2,2,2], k = 2_x000D_
#Output: 16_x000D_
#_x000D_
#_x000D_
# _x000D_
#Constraints:_x000D_
#_x000D_
#_x000D_
#	1 <= nums.length <= 50000_x000D_
#	1 <= nums[i] <= 10^5_x000D_
#	1 <= k <= nums.length_x000D_
class Solution(object):
    def numberOfSubarrays(self, nums, k):
        """
        :type nums: List[int]
        :type k: int
        :rtype: int
        """
        # Approach 1: Sliding Window
        # Time Complexity: O(N)
        # Space Complexity: O(1)
        # Runtime: 1004 ms, faster than 13.33% of Python online submissions for Count Number of Nice Subarrays.
        # Memory Usage: 18.9 MB, less than 6.67% of Python online submissions for Count Number of Nice Subarrays.
        ans, count, left = 0, 0, 0
        for right, num in enumerate(nums):
            if num % 2 == 1:
                count += 1
            while count > k:
                if nums[left] % 2 == 1:
                    count -= 1
                left += 1
            if count == k:
                temp = left
                while nums[temp] % 2 == 0:
                    temp += 1
                ans += temp - left + 1
        return ans


import os
import sys

from PIL import Image
from flask import Flask, render_template, request, url_for, redirect, flash, session, abort
from flask_admin import Admin
from flask_admin.contrib.sqla import ModelView
from sqlalchemy import func, or_
from sqlalchemy.exc import SQLAlchemyError
from werkzeug.utils import secure_filename

from config import MY_PROJECTS_DATABASE_NAME, CONTACTS_DATABASE_NAME, REVIEW_DATABASE_NAME, \
    MY_ACHIEVEMENTS_DATABASE_NAME, SITE_DATABASE_NAME
from db.models import db_init, db, Review, Project, Contact, Achievement
from utils import corvert_image

# <--------------------> #
#  InitializeBlockStart  #
# <--------------------> #

app = Flask(__name__, template_folder='templates', static_url_path='/static/')

# SQLAlchimy config
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + SITE_DATABASE_NAME
app.config['SQLALCHEMY_BINDS'] = {
    'contacts': 'sqlite:///' + CONTACTS_DATABASE_NAME,
    'reviews': 'sqlite:///' + REVIEW_DATABASE_NAME,
    'projects': 'sqlite:///' + MY_PROJECTS_DATABASE_NAME,
    'achievement': 'sqlite:///' + MY_ACHIEVEMENTS_DATABASE_NAME
}

app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = True
app.config['SECRET_KEY'] = '34567890oihgfcvbjktrtyuiop'
app.config['SESSION_COOKIE_NAME'] = 'EVM'
app.config['TIMEZONE'] = 'Europe/Paris'

# set optional bootswatch theme
app.config['FLASK_ADMIN_SWATCH'] = 'cerulean'

db.app = app
db.init_app(app)

# <------------------> #
#  InitializeBlockEnd  #
# <------------------> #

# <---------------> #
#  AdminBlockStart  #
# <---------------> #

class Controller(ModelView):
    def is_accessible(self):
        if "logged_in" in session:
            return True
        abort(403)

# Add administrative views here
admin = Admin(app, name='microblog', template_mode='bootstrap3')
admin.add_view(Controller(Review, db.session))
admin.add_view(Controller(Contact, db.session))
admin.add_view(Controller(Project, db.session))
admin.add_view(Controller(Achievement, db.session))

@app.route("/admin", methods=["POST"])
def photo():
    if request.args.get("f") == "add":
        file = request.files["photo"]
        path = os.getcwd() + url_for('static', filename=f'saved_images/{str(secure_filename(file.filename))}')
        img = Image.open(file.stream)
        img.save(path)
    if request.args.get("f") == "del":
        try:
            path = request.form.getlist('del')
            path_to_image = os.getcwd() + url_for('static', filename=f'saved_images/{path[0]}')
            print(path_to_image)
            os.remove(path_to_image)
        except:
            flash('something went wrong!', category='error')
    return redirect('/admin')

@app.route("/login", methods=['GET', 'POST'])
def login():
    if request.method == 'POST':
        if request.form.get("username") == "admin" and request.form.get("password") == "superqwerty":
            session['logged_in'] = True
            return redirect('/admin')
        else:
            return render_template('login.html', failed=True)
    return render_template('login.html')

@app.route("/logout")
def logout():
    session.clear()
    return redirect(url_for('index'))

# <--------------> #
#  AdminBlockEnd   #
# <--------------> #

# <---------------> #
#  RoutsBlockStart  #
# <---------------> #

@app.before_request
def make_permanent():
    session.permanent = True

@app.route("/")
@app.route("/index")
@app.route("/index.html")
def index():
    reviews = Review.query.filter(Review.visible).order_by(-Review.best).limit(10).all()
    return render_template('index.html',
                           title='Home',
                           reviews=reviews)

@app.route("/about")
@app.route("/about.html")
def about():
    return render_template('about_page.html', title="About")

@app.route("/projects")
@app.route("/projects.html")
def projects():
    projects = Project.query.all()
    reviews = Review.query.filter(Review.visible).order_by(Review.date).limit(5).all()
    return render_template('projects.html',
                           title="Projects",
                           projects=projects,
                           count=len(projects),
                           reviews=reviews)

@app.route("/projects/<int:id>")
@app.route("/projects_details.html/<int:id>")
def projects_details(id):
    project = Project.query.filter(Project.id == id).first()
    return render_template('projects_details.html', title="Details", project=project)

@app.route("/achievement", methods=['GET', 'POST'])
@app.route("/achievement.html", methods=['GET', 'POST'])
def achievement():
    if request.method == 'POST':
        try:
            achievements = Achievement.query.filter(or_(
                func.lower(Achievement.name).contains(request.form["res"]),
                func.lower(Achievement.categories).contains(request.form["res"]))).order_by(-Achievement.date_year,
                                                                                -Achievement.date_month,
                                                                                -Achievement.date_day).all()
            return render_template('achievement.html', title="Achievement", posts=achievements)
        except SQLAlchemyError as e:
            db.session.rollback()
            error = str(e.__dict__['orig'])
            print(error)
            print("Can't add new review to the Database")
            flash(error, category='error')
            return redirect(url_for('/achievement/all'))

    else:
        achievements = Achievement.query.order_by(-Achievement.date_year, -Achievement.date_month,
                                                  -Achievement.date_day).all()

    return render_template('achievement.html',
                           title="Achievement",
                           posts=achievements)

@app.route("/all_reviews", methods=['GET', 'POST'])
@app.route("/all_reviews.html", methods=['GET', 'POST'])
def all_reviews():
    best = None
    reviews = []
    projects = Project.query.limit(5).all()

    if request.method == 'POST':
        try:
            reviews = Review.query.filter(or_(
                func.lower(Review.name).contains(request.form["res"]),
                func.lower(Review.review).contains(request.form["res"]))).order_by(-Review.best, Review.date).all()
        except SQLAlchemyError as e:
            error = str(e.__dict__['orig'])
            print(error)
            print("Something wrong with the Database")
            flash(error, category='error')
            return redirect(url_for('index'))

    else:
        reviews = Review.query.filter(Review.visible).order_by(-Review.best, Review.date).all()

    count = len(reviews)
    if len(reviews) > 1:
        best, reviews = reviews[0], reviews[1:]
        count -= 1

    return render_template('all_reviews.html',
                           title="All reviews",
                           best=best,
                           reviews=reviews,
                           count=count,
                           projects=projects)

@app.route("/create_review", methods=['GET', 'POST'])
@app.route("/create_review.html", methods=['GET', 'POST'])
def create_review():
    if request.method == 'POST':
        try:
            file = request.files["photo"]
            name = f'src/default_image.png'

            if file.filename != '':
                filename = '.'.join(str(secure_filename(file.filename)).split('.')[:-1] + ['png'])
                name = f'saved_images/{filename}'
                path = url_for('static', filename=f'saved_images/{filename}')
                converted = corvert_image(file)
                converted.save(os.getcwd() + path)

            new_review = Review(name=request.form["name"] if request.form["name"] != '' else 'Anonim',
                                position=request.form["position"] if request.form["position"] != '' else 'Hidden',
                                review=request.form["review"] if request.form["review"] != '' else 'Good job!',
                                photo=name,
                                visible=True
                                )

            db.session.add(new_review)
            db.session.commit()
        except SQLAlchemyError as e:
            db.session.rollback()
            error = str(e.__dict__['orig'])
            print(error)
            print("Can't add new review to the Database")
            flash(error, category='error')
            return redirect(url_for('index'))

        flash("Thanks for your review. It will be added soon. Then you can check it in section: Review->All", category='success')
        return redirect(url_for('index'))
    else:
        return render_template('create_review.html', title="Create review")

@app.route("/contact", methods=['GET', 'POST'])
@app.route("/contact.html", methods=['GET', 'POST'])
def contact():
    if request.method == 'POST':
        try:
            new_contact = Contact(name=request.form["name"],
                                  email=request.form["email"],
                                  message=request.form["message"],
                                  subject=request.form["subject"]
                                  )
            db.session.add(new_contact)
            db.session.commit()

        except SQLAlchemyError as e:
            db.session.rollback()
            error = str(e.__dict__['orig'])
            print(error)
            print("Can't add contact to the Database")
            flash(error, category='error')
            return redirect(url_for('index'))

        flash("Thanks for contact with me. I will answer soon.", category='success')
        return redirect(url_for('index'))

    return render_template('contact.html', title="Contact")

# <-------------> #
#  RoutsBlockEnd  #
# <-------------> #

# Press the green button in the gutter to run the script.
if __name__ == '__main__':

    if len(sys.argv) > 1:
        if sys.argv[1] == "init":
            with app.app_context():
                db_init()

    app.run(host='0.0.0.0', port=5000, debug=True)


import re
class EmailErrorMsg:
    '''
    EmailErrorMsg
    '''
    msg_id = 0                      #email id
    content = ''                    #email content
    time = ''                       #email time and error time
    message_type = ''               #message type, can be error or warning
    error_type = ''                 #error type: cron job erro; Hadoop/Hive/Spark; Flink; Flume; Presto; Airflow; Promertheus; Jenkins;
    subject = ''                    #email subject
    digest_content = ''              #email error digest

    def __init__(self, id, msgcontent, subject, time):

        def findErrorType(self):
            if 'Cron <root@' in self.subject or 'PHP Fatal' in msgcontent:
                ret_err_type = 'Cron_job_failed'
            elif '[FIRING' in self.subject:
                ret_err_type = 'Promertheus'
            elif 'flink' in self.content:
                ret_err_type = 'Flink'
            elif 'flume' in self.content:
                ret_err_type = 'Flume'
            elif 'presto' in self.content:
                ret_err_type = 'Presto'
            elif 'jenkins' in self.content:
                ret_err_type = 'Jenkins'
            elif 'hadoop' in self.content or 'hive' in self.content or 'spark' in self.content:
                ret_err_type = 'Hadoop/Hive/Spark'
            else:
                ret_err_type = 'Airflow'
            return ret_err_type

        self.msg_id = id
        self.content = msgcontent
        self.subject = subject
        self.time = time
        self.error_type = findErrorType(self)
        if 'Cron <root@' in subject or 'PHP Fatal' in msgcontent:
            self.message_type = 'Error'
            self.digest_content = self.content.split('\n')[0]
        else:
            self.digest_content = subject
            if 'up_for_retry' in subject:
                self.message_type = 'Warning'
            else:
                self.message_type = 'Error'

    def display(self):
        print('| Id:{3} | Time:{0} | Type:{1}| Digest Content:{2} |'.format(self.time, self.message_type, self.digest_content, self.msg_id))

    def displayErrorContent(self):
        print('Id:{0} Error Content: {1}'.format(self.msg_id, self.content))

    def displayDigestContent(self):
        print('Id:{0} Digest Content: {1}'.format(self.msg_id, self.digest_content))

    def displayTime(self):
        print('Id:{0} Occur Time: {1}'.format(self.msg_id, self.time))

    def getOneSummary(self):
        return ' '+str(self.time)+' ||  '+ self.digest_content

    def getDigestContent(self):
        return re.sub(r'[^a-zA-Z0-9_: ]',r'',self.digest_content)

    def getErrorTime(self):
        return self.time.strftime('{%Y-%m-%d %H:%M:%S}')


import sys

input = sys.stdin.readline

dx = [1, -1, 0, 0]
dy = [0, 0, -1, 1]
row, col = 0, 0
result = 0

def dfs(graph, x, y, count, sum):
    global result
    global visited

    if count >= 4:
        result = max(result, sum)
        return

    for k in range(4):
        nx = x + dx[k]
        ny = y + dy[k]

        if 0 <= nx < row and 0 <= ny < col and not visited[nx][ny]:
            visited[nx][ny] = 1
            dfs(graph, nx, ny, count + 1, sum + graph[nx][ny])
            visited[nx][ny] = 0

def exception(graph, x, y):
    global result

    if 0 <= x + 2 < row and 0 <= y + 1 < col:
        value = graph[x][y] + graph[x + 1][y] + graph[x + 2][y] + graph[x + 1][y + 1]
        result = max(result, value)

    if 0 <= x + 1 < row and 0 <= y + 2 < col:
        value = graph[x][y] + graph[x][y + 1] + graph[x][y + 2] + graph[x + 1][y + 1]
        result = max(result, value)

    if 0 <= x - 1 and x + 1 < row and 0 <= y+1 < col:
        value = graph[x][y] + graph[x - 1][y + 1] + graph[x][y + 1] + graph[x + 1][y + 1]
        result = max(result, value)

    if 0 <= x - 1 and 0 <= y+2 < col:
        value = graph[x][y] + graph[x][y + 1] + graph[x][y + 2] + graph[x - 1][y + 1]
        result = max(result, value)

if __name__ == "__main__":
    row, col = map(int, input().split())

    graph = [list(map(int, input().split())) for _ in range(row)]

    visited = [[0] * col for _ in range(row)]

    for i in range(row):
        for j in range(col):
            visited[i][j] = 1
            dfs(graph, i, j, 1, graph[i][j])
            visited[i][j] = 0
            exception(graph, i, j)

    print(result)


rows = int(input())
matrix = []

for _ in range(rows):
    matrix.append(input())

symbol = input()

symbol_not_found = True
for row in range(rows):
    for col in range(rows):
        if matrix[row][col] == symbol:
            print(f"({row}, {col})")
            symbol_not_found = False

if symbol_not_found:
    print(f"{symbol} does not occur in the matrix")


import re

import requests

def get_url_from_user():
    url = input("enter url:")
    print("you entered %s" % url)
    return url

def get_webpage_from_url(url):
    webpage = requests.get(url)
    html_bytes = webpage.content
    html_string = html_bytes.decode('UTF-8')
    return html_string

def get_img_from_webpage(html_string):
    photo_match = re.findall("((http|https|ftp):\/\/[0-9a-zA-Z\"\'~%_,-]+[\/\.[0-9a-zA-Z%\"\'_,~\-]+?\.(jpeg|png|jpg|gif|mp4|webp))",html_string)
    #print(photo_match)
    for i, photo in enumerate(photo_match):
        pic_ptype = photo[2]
        print(photo)
        img_file = requests.get(photo[0])
        #print(img_file)
        with open('pictures_from_website/new_image_{n}.{ptype}'.format(n=i, ptype=pic_ptype),"wb") as f:
            f.write(img_file.content)

def main():
    url = get_url_from_user()
    webpage = get_webpage_from_url(url)
    get_img_from_webpage(webpage)
    #print("webpage")
    #print(webpage)

main()


from tkinter import *
import operator

list_widgets = []

#
# def sum_numbers():
#     sum_field_result.delete(0,END)
#
#     #check that fields are not letters and not empty
#
#
#     if sum_field_1.get() and sum_field_1.get().isnumeric() and (
#                 sum_field_1.get()[0] == '-' or not sum_field_1.get().isalpha()) and sum_field_2.get() and (
#                 sum_field_2.get().isnumeric() and (sum_field_2.get()[0] == '-' or not sum_field_2.get().isalpha())):
#
#         sum_field_result.insert(END, int(sum_field_1.get())+int(sum_field_2.get()))
#
#     else:
#         sum_field_result.insert(END, "unexpected input")
#
#
# def subtract_numbers():
#
#     subtract_field_result.delete(0, END)
#
#
#     if subtract_field_1.get() and subtract_field_1.get().isnumeric() and (
#             subtract_field_1.get()[0] == '-' or not subtract_field_1.get().isalpha()) and subtract_field_2.get() and (
#             subtract_field_2.get().isnumeric() and (subtract_field_2.get()[0] == '-' or not subtract_field_2.get().isalpha())):
#
#         subtract_field_result.insert(END, int(subtract_field_1.get()) - int(subtract_field_2.get()))
#
#     else:
#         subtract_field_result.insert(END, "unexpected input")
#
#
# def multiply_numbers():
#
#     multiply_field_result.delete(0, END)
#
#     if multiply_field_1.get() and multiply_field_1.get().isnumeric() and (
#             multiply_field_1.get()[0] == '-' or not multiply_field_1.get().isalpha()) and multiply_field_2.get() and (
#             multiply_field_2.get().isnumeric() and (
#             multiply_field_2.get()[0] == '-' or not multiply_field_2.get().isalpha())):
#
#         multiply_field_result.insert(END, int(multiply_field_1.get()) * int(multiply_field_2.get()))
#
#     else:
#         subtract_field_result.insert(END, "unexpected input")
#
#
# def divide_numbers():
#
#     divide_field_result.delete(0, END)
#
#     print(bool(divide_field_1.get() and divide_field_2.get()))
#
#     if divide_field_2.get() == "0":
#
#         divide_field_result.insert(END, "division by zero!")
#
#     elif divide_field_1.get() and divide_field_1.get().isnumeric() and (
#
#             divide_field_1.get()[0] == '-' or not divide_field_1.get().isalpha()) and divide_field_2.get() and (
#
#             divide_field_2.get().isnumeric() and (
#
#             divide_field_2.get()[0] == '-' or not divide_field_2.get().isalpha())):
#
#             divide_field_result.insert(END, int(divide_field_1.get()) / int(divide_field_2.get()))
#
#     else:
#         divide_field_result.insert(END, "unexpected input")

operators_list = ["+", "-", "*", "%"]

ops = {
    "+": operator.add,
    "-": operator.sub,
    "*": operator.mul,
    "%": operator.mod,
}

def calculate_result(operation, row):
    field_result.delete(0, END)
    field_result[row].insert(END, ops[operation](int(field_1.get(), int(field_2.get()))))

root = Tk()

for row, operation in enumerate(operators_list):
    Label(text=operation).grid(column=1, row=row)
    list_widgets.append(row)

    Button(text="=", command=lambda par_1=operation, par_2=list_widgets[row]: calculate_result(par_1, par_2)).grid(column=3, row=row)

for row in range(4):
    num_1 = StringVar()
    field_1 = Entry(textvariable=num_1)
    field_1.grid(column=0, row=row)

    num_2 = StringVar()
    field_2 = Entry(textvariable=num_2)
    field_2.grid(column=2, row=row)

    field_result = Entry()
    field_result.grid(column=4, row=row)

root.mainloop()

# from tkinter import *
#
#
# def print_entry():
#     print(entry_1.get())
#
#
# root = Tk()
#
# for row in range(3):
#     entry_1 = Entry()
#     entry_1.grid(column=0, row=row)
#
#     button_1 = Button(text=row, command=print_entry)
#     button_1.grid(column=1, row=row)
#
# root.mainloop()

# from tkinter import *
#
#
# def print_entry(text):
#     print(text)
#
#
# root = Tk()
#
# for row in range(3):
#     entry_1 = Entry()
#     entry_1.grid(column=0, row=row)
#
#     button_1 = Button(text=row, command=lambda widget=entry_1: print_entry(widget.get())) # save the widget not value (in lambda) but set the value
#     button_1.grid(column=1, row=row)
#
# root.mainloop()

# from tkinter import *
#
#
# def print_entry(text):
#     print(text)
#
#
# root = Tk()
#
# for row in range(3):
#     num_1 = StringVar()
#     entry_1 = Entry(textvariable=num_1)
#     entry_1.grid(column=0, row=row)
#
#     button_1 = Button(text=row, command=lambda value=num_1: print_entry(value.get())) # save the widget not value (in lambda) but set the value
#     button_1.grid(column=1, row=row)
#
# root.mainloop()

# from tkinter import *
#
# list_widgets = []
#
#
# def print_entry(text):
#     print(text)
#
#
# root = Tk()
#
# for row in range(3):
#     num_1 = StringVar()
#     entry_1 = Entry(textvariable=num_1)
#     entry_1.grid(column=0, row=row)
#     list_widgets.append(entry_1) #add widget to list
#
#     button_1 = Button(text=row, command=lambda value=list_widgets[row]: print_entry(value.get())) # save the widget not value (in lambda) but set the value
#     button_1.grid(column=1, row=row)
#
# root.mainloop()
#
#

MODULE 2/M2 Lesson 05 Calc Loops Lambda.py


import os
from padelpy import from_smiles
import pandas as pd
from util import load_environment_variables

def padel(smiles_string):
    # Compute descriptors for the given SMILES string
    return from_smiles(smiles_string)

def save_to_csv(descriptors, filepath, mode='w'):
    """Utility function to save descriptors to CSV."""
    df = pd.DataFrame(descriptors)
    df.to_csv(filepath, mode=mode, header=(mode == 'w'), index=False)

if __name__ == '__main__':
    # Load environment variables
    load_environment_variables()

    # Retrieve paths from environment variables
    DAT_PATH = os.getenv("DAT")
    RES_PATH = os.getenv("RES")

    # Extract SMILES string from 'Chromophore' column from the CSV file at DAT_PATH
    data = pd.read_csv(f'{DAT_PATH}/2023-10-02/solvents.csv')
    smiles = data['Solvent'].tolist()

    # Create empty lists for descriptors and errors
    descriptors = []
    error_descriptors = []

    for index, smile in enumerate(smiles):
        if index < 1:  # adjust this should you encounter another error
            continue
        try:
            descriptor = padel(smile)
            descriptor['Tag'] = index + 1
            descriptor['SMILES'] = smile
            descriptors.append(descriptor)
            print(f"Processed index: {index}")
        except Exception as e:
            print(f"Error processing SMILES string at index {index}: {smile}. Error: {e}")
            error_descriptors.append({"Tag": index + 1, "SMILES": smile, "Error": str(e)})

        # Save to file and clear cache every 50 processed entries
        if (index + 1) % 50 == 0:
            save_to_csv(descriptors, f'{RES_PATH}/2023-10-02/descriptors_solvent.csv', mode='a')
            descriptors.clear()
            print(f"Saved first {index+1} descriptors to file and cleared cache.")

    # Save any remaining descriptors and errors
    if descriptors:
        save_to_csv(descriptors, f'{RES_PATH}/2023-10-02/descriptors.csv', mode='a')
    if error_descriptors:
        pd.DataFrame(error_descriptors).to_csv(f'{RES_PATH}/2023-10-02/error_descriptors.csv', index=False)


import Code.Nanoparticle as NP
import Code.FCCLattice as FCC
import Code.CuttingPlaneUtilities as CPU
import Code.CutAndSpliceOperator as CPO
import Code.GlobalFeatureClassifier as GFC
import Code.MutationOperator as MO
import Code.PermutationOperator as PO
import Code.EnergyCalculator as EC
import Code.GA.Niched2DPopulation as NIP

from ase.visualize import view
import numpy as np
import matplotlib.pyplot as plt
import copy
import dill
import pickle
import sklearn.gaussian_process as gp
from scipy.spatial import ConvexHull

N_ATOMS = 79
symbols = ['Pt', 'Au']
lattice = FCC.FCCLattice(15, 15, 15, 2)

test_particle = NP.Nanoparticle(lattice)
test_particle.kozlov_sphere(5, symbols, [5, 74])
mixing_energy_calculator = EC.MixingEnergyCalculator()
mixing_energy_calculator.compute_mixing_parameters(test_particle, symbols)

feature_classifier = GFC.SimpleFeatureClassifier(symbols[0], symbols[1])

def initialize_start_population():
    start_population = NIP.Niched2DPopulation(79, 79, 'Pt')
    for n_pt_atoms in range(0, N_ATOMS + 1):
        niche = list()
        for i in range(1):
            p = NP.Nanoparticle(lattice)
            p.kozlov_sphere(5, symbols, [n_pt_atoms, N_ATOMS - n_pt_atoms])
            mixing_energy_calculator.compute_energy(p)
            feature_classifier.compute_feature_vector(p)
            start_population.add_offspring(p)

    return start_population

def locate_convex_hull(start_population, unsuccessful_gens_for_convergence, energy_calculator, energy_key):
    gens_since_last_success = 0

    mutation_operator = MO.MutationOperator(0.5)
    permutation_operator = PO.PermutationOperator(0.5)

    cutting_plane_generator = CPU.SphericalCuttingPlaneGenerator(0.0, 10.0)
    cut_and_splice_operator = CPO.CutAndSpliceOperator(cutting_plane_generator)

    population = copy.deepcopy(start_population)

    energy_log = dict()
    for n_pt_atoms in range(1, N_ATOMS):
        energy_log[n_pt_atoms] = [population[79][n_pt_atoms][0].get_energy(energy_key)]

    # feature_vector_log = dict()
    # for n_pt_atoms in range(1, N_ATOMS):
    # feature_vector_log[n_pt_atoms] = [population[n_pt_atoms][0].get_feature_vector()]

    cur_generation = 0
    while gens_since_last_success < unsuccessful_gens_for_convergence:
        cur_generation += 1
        print("Current generation: {0}".format(cur_generation))
        print(" ")
        print(['-' * 40])
        print(" ")

        if cur_generation % 200 == 0:
            pickle.dump(energy_log, open("energy_log.pkl", 'wb'))

        # choose the new particle from a priority region
        priority_compositions = list(range(N_ATOMS + 1))  # determine_priority_compositions(energy_log, 0.9)
        print("Priority: {0}".format(priority_compositions))

        p = np.random.random()
        if p < 0.6:
            # choose two parents for cut and splice
            print("Cut and Splice")

            while True:
                parents = population.gaussian_tournament(2, 5, (79, np.random.choice(priority_compositions, 1)[0]))
                parent1 = parents[0]
                parent2 = parents[1]
                new_particle = cut_and_splice_operator.cut_and_splice(parent1, parent2)

                if new_particle.get_stoichiometry()['Pt'] in priority_compositions:
                    break

        elif p < 0.8:
            # random permutation
            print("random permutation")
            while True:
                parent = population.random_selection(1)[0]
                new_particle = permutation_operator.random_permutation(parent)

                if new_particle.get_stoichiometry()['Pt'] in priority_compositions:
                    break

        else:
            # random mutation
            print("random mutation")
            while True:
                parent = population.gaussian_tournament(1, 5)[0]
                new_particle = mutation_operator.random_mutation(parent)

                if new_particle.get_stoichiometry()['Pt'] in priority_compositions:
                    break

        # check that it is not a pure particle
        new_stoichiometry = new_particle.get_stoichiometry()
        niche = new_stoichiometry['Pt']

        if niche == N_ATOMS or niche == 0:
            continue

        print("New Stoichiometry: {0}".format(new_stoichiometry))

        # check that it has a unique feature vector
        # feature_classifier.compute_feature_vector(new_particle)
        # unique = True
        # for feature_vector in feature_vector_log[niche]:
        # if np.array_equal(feature_vector, new_particle.get_feature_vector()):
        # unique = False
        # break

        # if not unique:
        # gens_since_last_success += 1
        # continue

        # feature_vector_log[niche].append(new_particle.get_feature_vector())

        # compute energy
        energy_calculator.compute_energy(new_particle)
        print("Energy: {0} = {1}".format(energy_key, new_particle.get_energy(energy_key)))

        # add new offspring to population
        successfull_offspring = False
        peers = population[79][niche]
        peers.sort(key=lambda x: x.get_energy(energy_key), reverse=True)
        if new_particle.get_energy(energy_key) - peers[-1].get_energy(energy_key) < -1e-6:
            print("success: {0}".format(new_particle.get_energy(energy_key) - peers[-1].get_energy(energy_key)))

            successfull_offspring = True
            peers.append(new_particle)

        population[79][niche] = peers

        # reset counters and log energy
        if successfull_offspring:
            gens_since_last_success = 0
            energy_log[niche].append(new_particle.get_energy(energy_key))
        else:
            gens_since_last_success += 1
            energy_log[niche].append(energy_log[niche][-1])

    return population, energy_log

if __name__ == '__main__':
    start_population = initialize_start_population()

    # %%capture
    n_convergence = 100

    n_runs = 1
    populations = list()
    logs = list()

    for run in range(n_runs):
        # start_population = initialize_start_population()
        # gpr_calculator = init_gpr(start_population)
        population, log = locate_convex_hull(start_population, n_convergence, mixing_energy_calculator, 'Mixing Energy')
        populations.append(population)
        logs.append(log)


from sklearn.linear_model import LogisticRegression
from sklearn.datasets import load_iris
from web_app.models import User , Tweet , db
# Will use the picke to load the file
import pickle
import os

#Path where the model will be saved
MODEL_PATH = os.path.join(os.path.dirname(__file__), os.path.pardir, "predictive_models", "saved_model.pkl")

# used to train the iris and save it
def train_model_and_save():
    print("Training the model")
    X, y = load_iris(return_X_y=True)
    classifier = LogisticRegression(random_state=49, max_iter=150)
    classifier.fit(X, y)

    # Saving the model
    with open(MODEL_PATH, "wb") as model:
        pickle.dump(classifier, model)

    return classifier

# Will return the trained model. Will read it to a
# file
def load_model():
    # Will load the model from the file
    with open(MODEL_PATH, "rb") as model:
        the_model = pickle.load(model)

    return the_model

# This is the method that will be used to train the
# for the twitter prediction
# The user1 and user2 are from the data base
def train_twitter_model(user1, user2):

    classifier = LogisticRegression(random_state=49, solver="lbfgs", multi_class="multinomial")
    # will get the embeddings and the screen name for the
    # two different  users
    embeddings = []
    userList = []

    tweets1 = user1.tweets
    tweets2 = user2.tweets

    # Getting the embedding from the tweets
    for tweet in tweets1:
        userList.append(user1.screen_name)
        embeddings.append(tweet.embedding)

    for tweet in tweets2:
        userList.append(user2.screen_name)
        embeddings.append(tweet.embedding)

    classifier.fit(embeddings, userList)
    # returning the model to do the prediction
    return classifier

if __name__ == "__main__":
    train_model_and_save()

    load_model()


# -*- coding: utf-8 -*-
from django.conf.urls import patterns, include, url
from django.contrib import admin
admin.autodiscover()

# import views
from asset.views import *
from opsa.views import *
from installed.views import *
from deploy.views import *
from config.views import *
from api.views import *
from message.views import *

urlpatterns = patterns('',
    # Examples:
    # url(r'^$', 'opsa.views.home', name='home'),
    # url(r'^blog/', include('blog.urls')),

    #url(r'^opsa/admin/', include(admin.site.urls)),
    #url(r'^login', login, name='login'),
    #url(r'^accounts/login/$', login, name='login'),
    url(r'^logout', logout, name='logout'),
    url(r'^$', index, name='index'),
    url(r'^asset/host_list/$', host_list, name='host_list'),
    url(r'^asset/add_host/$', host_list_manage, name='add_host'),
    url(r'^asset/delete_host/$', host_list_manage, name='delete_host'),
    url(r'^asset/host_manage/(?P<id>\d+)/$', host_list_manage, name='host_manage'),
    url(r'^asset/server_asset/$', server_asset_list, name='server_asset_list'),
    url(r'^asset/server_get/$', get_server_asset, name='get_server_asset'),
    url(r'^asset/grains_get/$', get_host_grains, name='get_host_grains'),
    url(r'^asset/hosts_autget/$', autoget_host_list, name='autoget_host_list'),
    url(r'^asset/device_list/$', network_device_list, name='network_device_list'),
    url(r'^asset/device_add/$', network_device_discovery, name='add_device'),
    url(r'^asset/idc_list/$', idc_asset_list, name='idc_asset_list'),
    url(r'^asset/add_idc/$', idc_asset_manage, name='add_idc'),
    url(r'^asset/listhostgroups/$', ListHostgroups, name='listhostgroups'),
    url(r'^asset/edithostgroups/(?P<ID>\d+)/$', EditHostgroups, name='edithostgroups'),
    url(r'^asset/deletehostgroups/(?P<ID>\d+)/$', DeleteHostgroups,name='deletehostgroups'),
    url(r'^asset/addhostgroups/$', AddHostgroups, name='addhostgroups'),
    url(r'^install/install_list/$', system_install_list, name='install_list'),
    url(r'^install/install_status/$', system_install_status, name='install_status'),
    url(r'^install/install_manage/(?P<id>\d+)/$', system_install_managed, name='install_manage'),
    url(r'^install/add_install/$', system_install_managed, name='add_install'),
    url(r'^install/install_logs/$', install_logs, name='install_logs'),
    url(r'^install/system_install/$',system_install, name='system_install'),
    url(r'^install/ipmi_c/$',ipmi_c, name='ipim_c'),
    url(r'^install/install_record/$',system_install_record, name='install_record'),
    url(r'^install/install_finish/$',system_install_finish, name='install_finish'),
    url(r'^install/install_init/$', deploy_init_status, name='install_init'),
    url(r'^deploy/job_list/$', salt_job_list, name='job_list'),
    url(r'^deploy/minions_status/$', host_list, name='minions_status'),
    url(r'^deploy/job_result/$', salt_job_results, name='job_result'),
    url(r'^deploy/key_list/$', salt_key_list, name='key_list'),
    url(r'^deploy/key_delete/$', salt_delete_key, name='delete_key'),
    url(r'^deploy/key_accept/$', salt_accept_key, name='accept_key'),
    url(r'^deploy/module_deploy/$', module_deploy, name='module_deploy'),
    url(r'^deploy/remote_execution/$', remote_execution, name='remote_execution'),
    url(r'^config/config_api_add/$', config_info, name='config_api_add'),
    url(r'^config/config_api_list/$', config_api_list, name='config_api_list'),
    url(r'^config/config_os_list/$', config_os_list, name='config_os_list'),
    url(r'^config/get_os_list/$', get_os_list, name='get_os_list'),
    url(r'^config/config_api_add/(?P<id>\d+)/$', config_info, name='config_api_add'),
    url(r'^api/install_status_collect/',install_status_collect, name='install_status_collect'),
    url(r'^message/mesg_list/',mesg_list, name='mesg_list'),
    url(r'^accounts/',include('UserManage.urls' )),
    url(r'^hosts/',include('hostmanage.urls' )),
)


import sys
sys.stdin = open('컨테이너 운반.txt','r')

# N개의 컨테이너를 M대의 트럭으로 운반
# 트럭당 한 개의 컨테이너를 운반할 수 있음, 적재량을 초과하면 운반 x
# M대의 트럭이 편도로 한번만 이동

# 테스트케이스
T = int(input())
for tc in range(1,T+1):
    # 컨테이너 수 N, 트럭 수 M
    N, M = map(int,input().split())
    # N개의 화물의 무게
    Wi = list(map(int,input().split()))
    # M개 트럭의 적재용량
    ti = list(map(int,input().split()))

    # 화물의 무게를 정렬
    Wi.sort(reverse=True)
    # 트럭의 적재 용량을 정렬
    ti.sort(reverse=True)
    # 화물의 무게를 나타냄
    weith = 0
    # 옮긴 총 무게를 나타내는 변수
    sum_weith = 0
    # 옮긴 횟수를 나타냄
    cnt = 0
    while True:
        # 가장 큰 무게가 적재 용량보다 작거나 같으면
        # 옮겼다는 의미, 트럭에서 1개를 뺀다.
        if Wi[weith] <= ti[0]:
            sum_weith += Wi[weith]
            ti.pop(0)
        weith += 1
        cnt += 1
        # 만약 하나도 못옮겼거나 화물이 남으면 반복문 종료
        if cnt >= M or weith >= N:
            break
    print(f'#{tc} {sum_weith}')


import sys
import os
from data.config import Config
from data.config import cfg
# 不要删除/home/zqr/data/test/下的目录
if __name__ == '__main__':
    # 用来整理整个流程
    # videosFolder 存放视频数据位置
    # imagesFolder 视频帧存放位置
    # opticalFolder 存放光流文件位置
    # dataInfoPath 数据文件存放位置
    # videosBboxPath 存放视频的bbox
    # imagesFolder_160 存放裁剪成160×160的视频帧
    # resultFolder 存放关键帧结果
    # optFilesFolder 存放原始视频光流图
    # optFilesFolder_160 存放resize成160×160光流图
    # if len(sys.argv) != 6:
    #     print("you should give enough param")
    #     sys.exit(1)
    # videosFolder = sys.argv[1]
    # imagesFolder = sys.argv[2]
    # opticalFolder = sys.argv[3]
    # dataInfoPath = sys.argv[4]
    # videosBboxPath = sys.argv[5]
    # imagesFolder_160 = sys.argv[6]
    # resultFolder = sys.argv[7]
    videosFolder = "/home/zqr/data/test/test_video"
    imagesFolder = "/home/zqr/data/test/test_img"
    opticalFolder = "/home/zqr/codes/data/opticalFiles"
    dataInfoPath = "/home/zqr/codes/data/data_info.txt"
    videosBboxPath = "/home/zqr/data/test/videosBbox.txt"
    imagesFolder_160 = "/home/zqr/codes/data/glofdb_frame_no_resize"
    resultFolder = "/home/zqr/codes/data/resultFolder"
    optFilesFolder_160 = "/home/zqr/codes/data/optFromOri_160"
    optFilesFolder = "/home/zqr/codes/data/optOri"

    # # step1: 切帧
    runStatus = os.system(
        'python3 getFrames.py')
    if runStatus != 0:
        print("切帧流程错误")
        sys.exit(1)
    print("切帧流程完成")

    # # step2：FPN 检测bouldingbox和球杆关键点
    runStatus = os.system('python3 getInfo.py')
    if runStatus != 0:
        print("检测候选框，球杆关键点，流程错误")
        sys.exit(2)
    print("检测候选框，球杆关键点，流程完成")

    # # step3: MSPN 检测人体关键点（候选框是人体候选框，目前主要是根据bbox大小来判断筛选出击球人）
    os.chdir("/home/zqr/codes/MSPN/exps/mspn.2xstg.coco")
    runStatus = os.system('python3 present.py')
    if runStatus != 0:
        print("检测人体关键点流程错误")
        sys.exit(3)
    print("检测人体关键点流程完成")

    # # # step4:整合关键点
    # os.chdir("/home/zqr/codes/MyGolfDB")
    # runStatus = os.system('python3 process_keypoints_data.py')
    # if runStatus != 0:
    #     print("整合关键点流程错误")
    #     sys.exit(2)
    # print("整合关键点流程完成")

    # # step3: 根据bbox 将图片重构为dim×dim(这一步待定吧)
    # os.chdir("/home/zqr/codes/MyGolfDB")
    # runStatus = os.system('python3 resizeFrame.py')
    # if runStatus != 0:
    #     print("resize流程错误")
    #     sys.exit(3)
    # print("resize流程完成")

    # step4: 光流法提取特征(flag是用来打开resize开关的，dim需要在光流proj的config里面配置一下，这样做的话就是在原图提光流特征，然后resize)
    # 这里没有完应该是根据bbox resize比较好吧
    os.chdir("/home/zqr/codes/pytorch-liteflownet")
    runStatus = os.system('python3 run.py {} {} {} {} {}'.format(
        cfg.TEST_IMGS_DIR, cfg.TEST_OPT_DIR, cfg.OPT_RESIZE_FLAG, cfg.TEST_BBOX_INFO_PATH,cfg.TEST_VIDEO_PAHT))
    if runStatus != 0:
        print("光流法提取特征错误")
        sys.exit(4)
    print("光流法提取特征完成")

    # step5: 提取关键帧(具体的路径都已经在disply_dataloader里面配置好了，直接使用就可以了)
    # 8帧
    os.chdir("/home/zqr/codes/MyGolfDB")
    runStatus = os.system(
        'python3 disply.py {}'.format(True))
    if runStatus != 0:
        print("关键帧8帧提取错误")
        sys.exit(5)
    print("关键帧8帧提取完成")

    # 13帧
    runStatus = os.system(
        'python3 disply.py {}'.format(False))
    if runStatus != 0:
        print("关键帧13帧提取错误")
        sys.exit(5)
    print("关键帧13帧提取完成")

    runStatus = os.system('python3 merge_result.py')
    if runStatus != 0:
        print("merge错误")
        sys.exit(5)
    print("merge完成")

    # step6:后处理功能

    # step7:在提取的关键帧图片上加关键点
    runStatus = os.system('python3 visualize.py')
    if runStatus != 0:
        print("13帧关键点错误")
        sys.exit(7)
    print("13帧关键点完成")

    # step8:动作评价


def open_file(file):
    file_op = open(file, 'r', encoding='utf-8')
    data = []
    for line in file_op.readlines():
        line = line.replace('\n', '').split('\t')
        data.append([line[0], int(line[1])])
    file_op.close()
    return data

def open_tsv(tsv):
    tsv_op = open(tsv, 'r', encoding='utf-8')
    prediction = []
    for line in tsv_op.readlines():
        line = line.replace('\n', '').split('\t')
        prediction.append(line.index(max(line)))
    # print(len(prediction))
    # class_sum(prediction)

    for i in range(len(prediction)):
        prediction[i] -= 1
    tsv_op.close()
    return prediction

def check(prediction, test_data):
    list = []
    for i in range(len(prediction)):
        if(prediction[i]!=test_data[i][1]):
            list.append(i)
    return list

test_file = '../data/cleaned_data/less/test.txt'
predict_file = './cleaned_data2/less/117/test_results.tsv'

prediction = open_tsv(predict_file)
test_data = open_file(test_file)

wrong_list = check(prediction, test_data)
print(wrong_list)
print(len(wrong_list))#1686
for i in wrong_list:
    print(test_data[i][0], prediction[i], test_data[i][1])