import urllib3
urllib3.disable_warnings()
import os.path

import json
import requests
import uservoice
import time

from config import *

def send_ticket_to_freshdesk(ticket, ticket_number):
	_url = "https://%s.freshdesk.com/api/v2/tickets" % FD_SUBDOMAIN
	_headers = {
		'Content-Type': 'application/json'
	}
	_data = json.dumps(ticket)

	time.sleep(1) # to prevent rate limit
	r = requests.post(_url, data = _data, auth=(FD_API_KEY, "api"), headers=_headers, verify=False)
	if r.status_code != 200 and r.status_code != 201:
		print('x %s: %s' % (ticket_number, r.text))
		return {}
	else:
		# print('✓ %s' % (ticket_number))
		return r.json()

def process_uv_ticket(uv_ticket):
	ticket = {}
	ticket['status'] = 5
	ticket['priority'] = 1
	ticket['source'] = 2
	ticket['email'] = uv_ticket['contact']['email']
	ticket['name'] = uv_ticket['contact']['name']
	description = "From UserVoice: %s
" % (
		UV_SUBDOMAIN,
		uv_ticket['ticket_number'],
		uv_ticket['ticket_number']
	)
	for message in uv_ticket['messages'][::-1]:
		if message['is_admin_response']:
			ticket['status'] = 4
		description += (
			"
---
"
			"%s <%s>
"
			"%s
"
			"%s
"
			"
%s

"
		) % (
			message['sender']['name'],
			message['sender']['email'],
			message['created_at'],
			message['referrer'],
			message['body'],
		)
	ticket['description'] = description
	ticket['subject'] = uv_ticket['subject']
	ticket['tags'] = ['uservoice']

	send_ticket_to_freshdesk(ticket, uv_ticket['ticket_number'])

def download_uv_tickets(uv_ticket_outfile, total_records):
	'''thanks https://github.com/drmarshall/zendesk_migration'''
	uv_client = uservoice.Client(UV_SUBDOMAIN, UV_API_KEY, UV_API_SECRET)
	base_url = "/api/v1/tickets.json?"
	total_records = total_records

	tickets = {}
	page = 1
	print("Downloading page %s" % str(page))
	with open(uv_ticket_outfile, "w") as f:
		while total_records > page*100:
			request = base_url+"page="+str(page)+"&per_page=100"+"&type=Support+Request&state=closed"
			response = uv_client.get(request)
			f.write(json.dumps(response['tickets'])+"\n")
			print(json.dumps(response['response_data']))
			page = response['response_data']['page'] + 1
			total_records = response['response_data']['total_records']
			print("Downloading page %s of %s pages" % (str(page), str(total_records/100)))

def import_tickets_to_freshdesk(uv_ticket_outfile):
	with open("uservoice_export.json", "r") as uservoice_export:
		batch = 1
		current = 0

		tickets = []
		total = 0
		for line in uservoice_export:
			ticket_batch = json.loads(line)
			for ticket in ticket_batch:
				tickets.append(ticket)
				total += 1

		for ticket in tickets:
			process_uv_ticket(ticket)
			current += 1
			print("%s/%s" % (current, total))

if __name__ == '__main__':
	uv_ticket_outfile = "uservoice_export.json"
	if not os.path.isfile(uv_ticket_outfile):
		download_uv_tickets(uv_ticket_outfile, total_records=60000)
	import_tickets_to_freshdesk(uv_ticket_outfile)


from typing import TYPE_CHECKING, Optional

from nonebot import Bot
from nonebot.internal.adapter import Event

from nonebot_plugin_access_control_api.subject.model import SubjectModel
from nonebot_plugin_access_control_api.subject.manager import SubjectManager

if TYPE_CHECKING:
    from nonebot.adapters.kaiheila.event import User
    from nonebot.adapters.kaiheila.event import Event as KaiheilaEvent

OFFER_BY = "nonebot_plugin_access_control"

def extract_kaiheila_role(bot: Bot, event: Event, manager: SubjectManager):
    if bot.type != "Kaiheila":
        return

    event: KaiheilaEvent

    guild_id: Optional[str] = event.extra.guild_id
    getattr(event, "group_id", None) or getattr(event.extra.body, "channel_id", None)
    author: Optional["User"] = event.extra.author

    if author is not None:
        li = []

        for role in sorted(author.roles):
            li.append(
                SubjectModel(
                    f"kaiheila:g{guild_id}.role_{role}", OFFER_BY, "kaiheila:guild.role"
                )
            )

        # 添加在platform:guild:channel之前
        manager.insert_before("platform:guild:channel", *li)


import lightgbm as lgb
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split

def rmsle(y_true, y_pred):
    assert len(y_true) == len(y_pred)
    return np.sqrt(np.mean(np.power(np.log1p(y_true + 1) - np.log1p(y_pred + 1), 2)))

def rmse(y_true, y_pred):
    assert len(y_true) == len(y_pred)
    return np.sqrt(np.mean(np.power(y_true - y_pred, 2)))

df = pd.read_csv('./data/feature_select_new.csv', header=0)
df_ans = pd.read_csv('./data/ans.csv', names=['ans'])
x_train, x_valid, y_train, y_valid = train_test_split(df, df_ans, test_size=0.10, random_state=7)

lgb_train = lgb.Dataset(x_train, y_train)
lgb_eval = lgb.Dataset(x_valid, y_valid,
                       # reference=lgb_train
                       )

model_param = {'lr': 0.005, 'depth': 10, 'tree': 500, 'leaf': 10, 'sample': 0.9, 'seed': 3}
params = {
    'task': 'train',
    'boosting_type': 'gbdt',
    'objective': 'regression_l2',
    'metric': {'l2', 'l1'},
    'max_depth': model_param['depth'],
    'num_leaves': model_param['leaf'],
    'min_data_in_leaf': 20,
    'learning_rate': model_param['lr'],
    'feature_fraction': 1,
    'bagging_fraction': model_param['sample'],
    'bagging_freq': 1,
    'bagging_seed': model_param['seed'],
    'verbose': 0,
    'min_data_in_leaf': 1,
    'min_data_in_bin': 1
}

model = lgb.train(params, lgb_train, num_boost_round=5000, #tree
                  valid_sets=lgb_eval,
                  feval=rmse,
                  early_stopping_rounds=30,
                  verbose_eval=True
                  )

preds = model.predict(x_valid)
print(rmse(y, preds))
#
# # print("調參1：提高準確率")
# min_merror = float('Inf')
# for num_leaves in range(20, 450, 100):
#     for max_depth in range(5, 16, 5):
#         params['num_leaves'] = num_leaves
#         params['max_depth'] = max_depth
#         cv_results = lgb.cv(
#             params,
#             lgb_train,
#             seed=42,
#             nfold=3,
#             early_stopping_rounds=10,
#             verbose_eval=True
#         )
#         mean_merror = pd.Series(cv_results['l2-mean']).min()
#         if mean_merror < min_merror:
#             min_merror = mean_merror
#             best_params['num_leaves'] = num_leaves
#             best_params['max_depth'] = max_depth
# params['num_leaves'] = best_params['num_leaves']
# params['max_depth'] = best_params['max_depth']
# # overfitting
# print("調参2：降低overfit")
# for max_bin in range(1, 155, 25):
#     for min_data_in_leaf in range(10, 101, 10):
#         params['max_bin'] = max_bin
#         params['min_data_in_leaf'] = min_data_in_leaf
#
#         cv_results = lgb.cv(
#             params,
#             lgb_train,
#             seed=42,
#             nfold=3,
#             early_stopping_rounds=3,
#             # verbose_eval=True
#         )
#
#         mean_merror = pd.Series(cv_results['l2-mean']).min()
#
#         if mean_merror < min_merror:
#             min_merror = mean_merror
#             best_params['max_bin'] = max_bin
#             best_params['min_data_in_leaf'] = min_data_in_leaf
#
# params['min_data_in_leaf'] = best_params['min_data_in_leaf']
# params['max_bin'] = best_params['max_bin']
# print params
# print("調参3：降低overfit")
# for feature_fraction in [i / 10.0 for i in range(0, 11, 2)]:
#     for bagging_fraction in [i / 10.0 for i in range(0, 11, 2)]:
#         for bagging_freq in range(0, 50, 5):
#             params['feature_fraction'] = feature_fraction
#             params['bagging_fraction'] = bagging_fraction
#             params['bagging_freq'] = bagging_freq
#
#             cv_results = lgb.cv(
#                 params,
#                 lgb_train,
#                 seed=42,
#                 nfold=3,
#                 early_stopping_rounds=3,
#                 # verbose_eval=True
#             )
#
#             mean_merror = pd.Series(cv_results['l2-mean']).min()
#             boost_rounds = pd.Series(cv_results['l2-mean']).argmin()
#
#             if mean_merror < min_merror:
#                 min_merror = mean_merror
#                 best_params['feature_fraction'] = feature_fraction
#                 best_params['bagging_fraction'] = bagging_fraction
#                 best_params['bagging_freq'] = bagging_freq
#
# params['feature_fraction'] = best_params['feature_fraction']
# params['bagging_fraction'] = best_params['bagging_fraction']
# params['bagging_freq'] = best_params['bagging_freq']
# print params
# print("調参4：降低overfit")
# for lambda_l1 in [i / 10.0 for i in range(0, 11, 2)]:
#     for lambda_l2 in [i / 10.0 for i in range(0, 11, 2)]:
#         for min_split_gain in [i / 10.0 for i in range(0, 11, 2)]:
#             params['lambda_l1'] = lambda_l1
#             params['lambda_l2'] = lambda_l2
#             params['min_split_gain'] = min_split_gain
#
#             cv_results = lgb.cv(
#                 params,
#                 lgb_train,
#                 seed=42,
#                 nfold=3,
#                 early_stopping_rounds=3,
#                 # verbose_eval=True
#             )
#
#             mean_merror = pd.Series(cv_results['l2-mean']).min()
#
#             if mean_merror < min_merror:
#                 min_merror = mean_merror
#                 best_params['lambda_l1'] = lambda_l1
#                 best_params['lambda_l2'] = lambda_l2
#                 best_params['min_split_gain'] = min_split_gain
#
# params['lambda_l1'] = best_params['lambda_l1']
# params['lambda_l2'] = best_params['lambda_l2']
# params['min_split_gain'] = best_params['min_split_gain']
#
# print(model_param)
# print(params)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# {# pkglts, pysetup.kwds
# format setup arguments

import os
from os import walk
from os.path import abspath, normpath, dirname
from os.path import join as pj

from setuptools import setup, find_packages

short_descr = "Python/Visualea interface to Caribu Light model"
readme = open('README.rst').read()
history = open('HISTORY.rst').read().replace('.. :changelog:', '')

# find version number in src/alinea/caribu/version.py
version = {}
with open("src/alinea/caribu/version.py") as fp:
    exec(fp.read(), version)
version_caribu = version["__version__"]

data_files = []

nb = len(normpath(abspath("src/caribu_data"))) + 1

def data_rel_pth(pth):
    """ Return path relative to pkg_data
    """
    abs_pth = normpath(abspath(pth))
    return abs_pth[nb:]

"""
for root, dnames, fnames in walk("src/caribu_data"):
    for name in fnames:
        data_files.append(data_rel_pth(pj(root, name)))
"""

setup_kwds = dict(
    name='alinea.caribu',
    version=version_caribu,
    description=short_descr,
    long_description=readme + '\n\n' + history,
    author="Christian Fournier, Michael Chelle, Christophe Pradal ",
    author_email="Christian.Fournier@supagro.inra.fr, michael.chelle@grignon.inra.fr, christophe dot pradal _at_ cirad fr ",
    url='https://github.com/openalea-incubator/caribu',
    license='INRA_License_agreement',
    zip_safe=False,

    packages=find_packages('src'),
    package_dir={'': 'src'},

    include_package_data=True,
    package_data={},
    entry_points={},
    keywords='openalea, FSPM, light',
    #test_suite='nose.collector',
)
# #}
# change setup_kwds below before the next pkglts tag

#setup_kwds['setup_requires'] = ['openalea.deploy']

build_prefix = pj(abspath(dirname(__file__)),"build-scons")
setup_kwds['scons_scripts'] = ['SConstruct']
setup_kwds['bin_dirs'] = {'bin': build_prefix + '/bin'}
setup_kwds['lib_dirs'] = {'lib' : build_prefix+'/lib' }
setup_kwds['inc_dirs'] = { 'include' : build_prefix+'/include' }
setup_kwds['entry_points']['wralea'] = ['alinea.caribu = alinea.caribu_wralea']
#setup_kwds['entry_points']['console_scripts'] = []
setup_kwds['package_data'][''] = ['*.can', '*.R', '*.8', '*.opt', '*.light', '*.csv', '*.png']

if ('CONDA_PREFIX' not in os.environ) and ('PREFIX' not in os.environ):
    setup_kwds['namespace_packages']=['alinea'] # Never used in a conda env...

setup_kwds['setup_requires'] = ['openalea.deploy']

# do not change things below
# {# pkglts, pysetup.call
setup(**setup_kwds)
# #}


import sys
from math import log, sqrt
from itertools import combinations
import json
from nltk.stem.porter import *
from nltk.stem.snowball import ArabicStemmer
from string import punctuation

punctuation += '،؛؟”0123456789“'

stopWords = open(".../arabic_stopwords.txt",encoding = "utf-8").read().splitlines()

words=" ".join(stopWords)
STOPWORDS = frozenset({ w for w in words.split() if w })

def remove_stopword(sentence):
    return " ".join(w for w in sentence.split() if w not in STOPWORDS)

def snowballStemmer(word):
    stemmer = ArabicStemmer()
    stem=stemmer.stem(word)
    return stem

def cosine_distance(a, b):
    cos = 0.0
    a_tfidf = a["tfidf"]
    for token, tfidf in b["tfidf"].items():
        if token in a_tfidf:
            cos += tfidf * a_tfidf[token]
    return cos

def normalize(features):
    if features != {}:
        x=sqrt(sum(i**2 for i in features.values()))
        if x !=0:
            norm = 1.0 / x
            for k, v in features.items():
                features[k] = v * norm
    return features

def add_tfidf_to(documents):
    tokens = {}
    for id, doc in enumerate(documents):
        tf = {}
        doc["tfidf"] = {}
        doc_tokens = doc.get("tokens", [])
        for token in doc_tokens:
            tf[token] = tf.get(token, 0) + 1
        num_tokens = len(doc_tokens)
        if num_tokens > 0:
            for token, freq in tf.items():
                tokens.setdefault(token, []).append((id, float(freq) / num_tokens))

    doc_count = float(len(documents))
    for token, docs in tokens.items():
        idf = log(doc_count / len(docs))
        for id, tf in docs:
            tfidf = tf * idf
            if tfidf > 0:
                documents[id]["tfidf"][token] = tfidf

    for doc in documents:
        doc["tfidf"] = normalize(doc["tfidf"])

def choose_cluster(node, cluster_lookup, edges):
    new = cluster_lookup[node]
    if node in edges:
        seen, num_seen = {}, {}
        for target, weight in edges.get(node, []):
            seen[cluster_lookup[target]] = seen.get(
                cluster_lookup[target], 0.0) + weight
        for k, v in seen.items():
            num_seen.setdefault(v, []).append(k)
        new = num_seen[max(num_seen)][0]
    return new

def majorclust(graph):
    cluster_lookup = dict((node, i) for i, node in enumerate(graph.nodes))

    count = 0
    movements = set()
    finished = False
    while not finished:
        finished = True
        for node in graph.nodes:
            new = choose_cluster(node, cluster_lookup, graph.edges)
            move = (node, cluster_lookup[node], new)
            if new != cluster_lookup[node] and move not in movements:
                movements.add(move)
                cluster_lookup[node] = new
                finished = False

    clusters = {}
    for k, v in cluster_lookup.items():
        clusters.setdefault(v, []).append(k)

    return clusters.values()

def get_distance_graph(documents):
    class Graph(object):
        def __init__(self):
            self.edges = {}

        def add_edge(self, n1, n2, w):
            if w>=0.0:#mus added
                self.edges.setdefault(n1, []).append((n2, w))
                self.edges.setdefault(n2, []).append((n1, w))

    graph = Graph()
    doc_ids = range(len(documents))
    graph.nodes = set(doc_ids)
    for a, b in combinations(doc_ids, 2):
        graph.add_edge(a, b, cosine_distance(documents[a], documents[b]))
    return graph

def get_documents(string,ii):
    import sumy
    from sumy.parsers.plaintext import PlaintextParser
    from sumy.nlp.tokenizers import Tokenizer
    LANGUAGE="arabic"
    parser= PlaintextParser(string, Tokenizer(LANGUAGE)).document.sentences

    raw={}
    for s_no, sent in enumerate(parser):
        raw[s_no]=str(sent)

    with open(".../raw"+str(ii)+".json", "w",encoding = "utf-8") as write_file:
        json.dump(raw, write_file)
    write_file.close()

    li=[]
    for i in parser:
         i=str(i)
         line = ''.join(c for c in i if c not in punctuation)
         line= remove_stopword(line)
         line=line.split()
         line=[ snowballStemmer(word) for word in line]#SnowballStemmerx
         li.append(line)
    docs=li
    return [{"s_id":s_id,"text": " ".join(text), "tokens": text} for s_id, text in enumerate(docs)]

'''Main program ...............................'''
def main(args):
    data_x={}
    for ii in range(1,154):
        r_text=open(".../document"+str(ii)+".txt",'r', encoding='utf-8').read()
        documents = get_documents(r_text,ii)
        add_tfidf_to(documents)
        senID={}
        for i in documents:
            senID[i['s_id']]=i['tfidf']

        with open(".../tfidf"+str(ii)+".json", "w",encoding = "utf-8") as write_file:
            json.dump(senID, write_file)
        write_file.close()

        dist_graph = get_distance_graph(documents)
        edges=dist_graph.edges

        s={}
        for k,result in list(edges.items()):
            result.sort(key=lambda x: x[1])
            result.reverse()
            dn=[]
            for i in result:
                if i[1] != 0.0:
                    dn.append(i)

            ldn=len(dn)

            s[int(k)]=(result,ldn/len(documents))

        with open(".../nearest"+str(ii)+".json", "w",encoding = "utf-8") as write_file:
            json.dump(s, write_file)
        write_file.close()

        print("The end......(",ii,")")

if __name__ == '__main__':
    main(sys.argv)


import numpy as np
import matplotlib.pyplot as plt
import copy

from run_stages.common_run_stage import CommonRunStage
from configuration.configuration_manager import Configuration
from configuration.models import Models
from utilities.image_processing_utilities import int_sq

class ResistiveMeshStage(CommonRunStage):
	"""
	This class implements the logic for the resistive mesh stage while abiding by the structure required by the
	common run stage
	"""

	def __init__(self, *args):
		super().__init__(*args)
		self.resistive_mesh = None

	def __str__(self):
		return "Resistive Mesh Stage"

	@property
	def stage_name(self):
		return "resistive_mesh"

	@property
	def output_file_name(self):
		return [Configuration().params["r_matrix_output_file"]]

	def run_stage(self):
		"""
		This function holds the execution logic for the resistive mesh stage
		:param args:
		:param kwargs:
		:return:
		"""
		self.resistive_mesh = self._build_interconnected_mesh()
		# self.plot_resistive_mesh()
		return self.resistive_mesh

	@staticmethod
	def _build_interconnected_mesh():
		"""
		This function is the heart of this stage.
		It converts the R matrix into a symmetric interconnected mesh of resistors
		:return:
		"""
		if Configuration().params["model"] == Models.MONOPOLAR.value:
			# load the non-diagonal entries of the R matrix. The first 3 columns are coordinates
			R = np.loadtxt(Configuration().params["r_matrix_input_file_non_diagonal"], delimiter=',')

			# load the diagonal entries of the R matrix. The first 2 columns are coordinates
			R_dia = np.loadtxt(Configuration().params["r_matrix_input_file_diagonal"], delimiter=',')

			# All potentials are referenced to the return mesh of the elementary field (column 3 of R_dia)
			R = R[:, 3:] - R_dia[:, 3]
			# Column 2 of R_dia contains the average potential on each active electrode.
			R_dia = R_dia[:, 2] - R_dia[:, 3]
			np.fill_diagonal(R, R_dia)

		if Configuration().params["model"] == Models.BIPOLAR.value:
			# load the axially symmetric field generated by a single active electrode
			dat_active = np.loadtxt(Configuration().params["r_matrix_input_file_active"], delimiter=',')
			# load the  axially symmetric far field generated by a single return electrode
			dat_return = np.loadtxt(Configuration().params["r_matrix_input_file_return"], delimiter=',')
			# load the near field generated by a single return electrode
			dat_return_neighbor = np.loadtxt(Configuration().params["r_matrix_input_file_return_neighbor"], delimiter=',')
			# load the coordinates of all pixels
			px_pos = np.loadtxt(Configuration().params["r_matrix_input_file_px_pos"], delimiter=',')

			active_r = Configuration().params["active_electrode_radius"]
			return_w = Configuration().params["return_width"]
			px_size = Configuration().params["pixel_size"]
			N_px = px_pos.shape[0]

			# distance and potential from the center of an active electrode
			X_act = dat_active[0,:]
			V_act = dat_active[1,:]
			# distance and potential from the center of a return hexagon
			X_ret = dat_return[0,:]
			V_ret = dat_return[1,:]
			# determine the self-resistance of the active
			X_idx = X_act<active_r
			Rself_act = int_sq(X_act[X_idx], V_act[X_idx])
			# cross-pixel resistance from the return to the active of the same pixel
			Rself_ra = int_sq(X_ret[X_idx], V_ret[X_idx])
			# self-resistance of an return hexagon
			Rself_ret = dat_return_neighbor[0]
			# cross-pixel resistance from the return to the active of the same pixel
			return_r = px_size*(.5+np.tan(np.pi/6))/2
			X_idx = (X_ret<=return_r)&(X_ret>return_r-return_w)
			Rself_ar = int_sq(X_act[X_idx], V_act[X_idx])

			# interprelate far-field entries
			x_dist = px_pos[:,0].reshape((N_px, 1)) - px_pos[:,0].reshape((1, N_px))
			y_dist = px_pos[:,1].reshape((N_px, 1)) - px_pos[:,1].reshape((1, N_px))
			dist = np.sqrt(x_dist**2 + y_dist**2)
			R_act =  np.interp(dist, X_act, V_act)
			R_ret =  np.interp(dist, X_ret, V_ret)

			# cross-pixel resistance to a return hexagon from the 3 nearest neighbor returns
			neighbor1_idx = np.abs(dist - px_size)<1E-3
			neighbor2_idx = np.abs(dist - 2*px_size)<1E-3
			neighbor3_idx = np.abs(dist - np.sqrt(3)*px_size)<1E-3
			R_ret[neighbor1_idx] = dat_return_neighbor[1]
			R_ret[neighbor2_idx] = dat_return_neighbor[2]
			R_ret[neighbor3_idx] = dat_return_neighbor[3]

			# cross-pixel resistance between active and return
			R12 = copy.deepcopy(R_act)
			R21 = copy.deepcopy(R_ret)
			np.fill_diagonal(R12, Rself_ar*np.ones(N_px))
			np.fill_diagonal(R21, Rself_ra*np.ones(N_px))
			np.fill_diagonal(R_act, Rself_act*np.ones(N_px))
			np.fill_diagonal(R_ret, Rself_ret*np.ones(N_px))

			# 2Nx2N impedance matrix
			R = np.concatenate((np.concatenate((R_act, R12)), np.concatenate((R21, R_ret))), axis=1)

		# force symmetry
		R = (R + R.T) / 2
		# convert the impedance matrix to resistor mesh. see [Z.C. Chen, et al, 2022] for the theory.
		G = np.linalg.inv(R)
		S = - G
		np.fill_diagonal(S, G.sum(axis=1))
		r_dual = 1 / S

		return r_dual / Configuration().params["r_matrix_conductivity"] * 1E3

	def plot_resistive_mesh(self):
		# Just for visualization of the resistors in the mesh. optional
		plt.plot(np.diag(self.resistive_mesh * 1E-6), label='$R_{n,n}$')
		plt.xlabel('Pixel Index')
		plt.ylabel('Resistance (M$\Omega$)')
		plt.legend()


import connexion
import six

from swagger_server.models.meta_page import MetaPage  # noqa: E501
from swagger_server.models.page import Page  # noqa: E501
from swagger_server import util, const
from swagger_server.database import database
from swagger_server.controllers.exceptions import ExceptionHandler
from swagger_server.controllers import subpage_controller, layout_controller, social_controller

def create_page(login, pasword):  # noqa: E501
    """creates a new page

    creates a blank project with one blank subpage - requires creating new account (user must provide login and password) # noqa: E501

    :param login:
    :type login: str
    :param pasword:
    :type pasword: str

    :rtype: Page
    """
    return 'do some magic!'

def get_page(id_page):  # noqa: E501
    """returns the requested page

    returns page that user/admin sent request for # noqa: E501

    :param id: id of page
    :type id: int

    :rtype: Page
    """
    id = const.DEFAULT_USER
    curr = database.conn.cursor(dictionary=True, buffered=True)
    curr.execute(f"SELECT * FROM Pages WHERE id = {id}",)
    a = curr.fetchone()
    if a is None:
        raise ExceptionHandler.NotFoundException

    curr.execute(f"SELECT * FROM MetaPages WHERE page = {id}")

    meta_query= curr.fetchone()
    curr.close()
    if meta_query is None:
        raise connexion.exceptions.ConnexionException()
    meta = MetaPage.from_dict(meta_query)
    layout = layout_controller.get_layout()[0]
    socials = social_controller.get_socials()
    subpages = subpage_controller.get_subpage_array()

    return Page(id=id, meta=meta,layout=layout,socials=socials, subpages=subpages)

def patch_meta_page(body):  # noqa: E501
    """modifies a metadata of a page

     # noqa: E501

    :param body: Meta information of page
    :type body: dict | bytes

    :rtype: MetaPage
    """
    if connexion.request.is_json:
        body = connexion.request.get_json()  # noqa: E501
    return 'do some magic!'


# get chat completions by by passing one or more messages to the chat model

from langchain.chat_models import ChatOpenAI  # import chat model
from langchain.schema import (
    AIMessage, HumanMessage, SystemMessage)  # type of message
from dotenv import dotenv_values

env_val = dotenv_values(".env")
openai_api_key = env_val["openai_api_key"]

chat = ChatOpenAI(openai_api_key=openai_api_key,
                  temperature=0)  # init chat model

# get complition by passing a single message.
single_res = chat([HumanMessage(
    content="Translate this sentence from English to Amharic. I love programing.")])

# print(single_res)

# multiple messages
messages = [
    SystemMessage(
        content="Your are a help full assistant that translates English to Amharic"),
    HumanMessage(content="whos pen is this ?")
]

multiple_message = chat(messages)

print(multiple_message)


import json
import os

SEQUENCE_LENGTH = 64
DATASET_PATH = './dataset'
SINGLE_FILE_DATASET_PATH = 'file_dataset'
MAPPING_PATH = 'mapping.json'

def load(file_path):
    with open(file_path, 'r') as file:
        song = file.read()
    return song

def save(songs, file_dataset_path):
    with open(file_dataset_path, 'w') as file:
        file.write(songs)

def create_single_file_dataset(dataset_path, file_dataset_path, sequence_length):
    new_song_delimiter = '/ ' * sequence_length
    songs = ""
    for path, _, files in os.walk(dataset_path):
        for file in files:
            file_path = os.path.join(path, file)
            song = load(file_path)
            songs = songs + song + " " + new_song_delimiter
    songs = songs[:-1]
    save(songs, file_dataset_path)
    return songs

def create_mapping(songs, mapping_path):
    mappings = {}
    songs = songs.split()
    vocabulary = list(set(songs))
    for i, symbol in enumerate(vocabulary):
        mappings[symbol] = i
    with open(mapping_path, 'w') as file:
        json.dump(mappings, file, indent=4)

def convert_into_int(songs):
    int_songs = []

    with open(MAPPING_PATH, 'r') as file:
        mappings = json.load(file)

    songs = songs.split()
    for symbol in songs:
        int_songs.append(mappings[symbol])

    return int_songs

def generate_training_data(sequence_length):
    songs = load(SINGLE_FILE_DATASET_PATH)
    int_songs = convert_into_int(songs)

    inputs = []
    targets = []

    num_sequences = len(int_songs) - sequence_length
    for i in range(num_sequences):
        inputs.append(int_songs[i:sequence_length+i])
        targets.append(int_songs[sequence_length+i])

    print(f'there are {num_sequences} of training data')
    return inputs, targets

if __name__ == '__main__':
    songs = create_single_file_dataset(DATASET_PATH, SINGLE_FILE_DATASET_PATH, SEQUENCE_LENGTH)
    create_mapping(songs, MAPPING_PATH)
    inputs, targets = generate_training_data(SEQUENCE_LENGTH)


"""
Creates a single-page HTTP server
"""
import os

from flask import Flask, request

def create_app(test_config=None):
    # create and configure the app
    app = Flask(__name__, instance_relative_config=True)
    app.config.from_mapping(
        SECRET_KEY='dev',
        DATABASE=os.path.join(app.instance_path, 'flaskr.sqlite'),
    )

    if test_config is None:
        # load the instance config, if it exists, when not testing
        app.config.from_pyfile('config.py', silent=True)
    else:
        # load the test config if passed in
        app.config.from_mapping(test_config)

    # ensure the instance folder exists
    try:
        os.makedirs(app.instance_path)
    except OSError:
        pass

    # a simple page that says hello
    @app.route('/', methods=['GET'])
    def hello():
        return 'HELLO'

    # a simple page that says hello
    @app.route('/', methods=['PUT', 'POST'])
    def update():
        print(request.data)
        return request.data

    app.run(debug=True, use_reloader=False)


import math

from numpy import ndarray, array
from scipy.spatial.transform import Rotation

from .GeoShape import GeoShape
from src.pyLiveKML.KML.GeoCoordinates import GeoCoordinates
from src.pyLiveKML.KML.KML import AltitudeMode

def ellipse_gen(
    x_rad: float, y_rad: float, rotation: Rotation = None, num_v: int = 32
) -> ndarray:
    step = 2 * math.pi / num_v
    angle = -step
    for i in range(0, num_v):
        angle += step
        point = array([x_rad * math.cos(angle), y_rad * math.sin(angle), 0])
        if rotation is not None:
            point = rotation.apply(point)
        yield point

class GeoEllipse(GeoShape):
    def __init__(
        self,
        origin: GeoCoordinates,
        x_radius: float,
        y_radius: float,
        rotation: Rotation = None,
        num_vertices: int = 32,
        border_width: float = 1.0,
        border_color: int = 0xFFFFFFFF,
        fill_color: int = 0xFFFFFFFF,
        name: str = None,
        selected: bool = False,
        altitude_mode: AltitudeMode = AltitudeMode.CLAMP_TO_GROUND,
    ):
        GeoShape.__init__(
            self,
            origin=origin,
            outer_bound=list(ellipse_gen(x_radius, y_radius, rotation, num_vertices)),
            inner_bounds=None,
            border_width=border_width,
            border_color=border_color,
            fill_color=fill_color,
            name=name,
            selected=selected,
            altitude_mode=altitude_mode,
        )


from collections import Counter
from collections import defaultdict

from nltk.corpus import stopwords
from wordcloud import WordCloud
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns

from preprocess import process_text_data
from utils import load_epub

def get_word_frequency_information(book):
    book_bow = defaultdict(list)

    stopword = stopwords.words('english')

    for chapter, text in book.items():
        for sentence in text:
            for word in sentence.split():
                if word in stopword:
                    continue
                book_bow[chapter].append(word)

    chapters = list()
    words = list()
    frequencies = list()

    for chapter, bow in book_bow.items():
        counter = Counter(bow)
        for word, frequency in counter.most_common():
            chapters.append(chapter)
            words.append(word)
            frequencies.append(frequency)
    word_frequency_information = pd.DataFrame(
        zip(
            chapters,
            words,
            frequencies
        ),
        columns=['chapter', 'word', 'frequency']
    )
    word_frequency_information.to_csv(
        './output/dataframes/q2-word_frequency_information.csv',
        index=False, encoding='utf-8'
    )

    return word_frequency_information

def get_top_word_frequency_information(word_freq_info, topn=100):
    words = word_freq_info.word.unique()
    frequencies =\
        word_freq_info.groupby(by='word').sum().loc[words, 'frequency'].values

    top_word_freq_info = pd.DataFrame(
        zip(
            words,
            frequencies
        ),
        columns=['word', 'frequency']
    )

    top_word_freq_info = top_word_freq_info.sort_values(
        by='frequency', ascending=False, ignore_index=True
    )
    top_word_freq_info = top_word_freq_info.head(topn)

    top_word_freq_info.to_csv(
        './output/dataframes/q2-top_word_frequency_information.csv',
        index=False, encoding='utf-8'
    )
    return top_word_freq_info

def draw_barplot_top_word_frequency_information(top_word_freq_info, badwords):
    badword_index =\
        top_word_freq_info[top_word_freq_info.word.isin(badwords)].index
    non_badword_index =\
        top_word_freq_info[~top_word_freq_info.word.isin(badwords)].index

    plt.figure(figsize=(12, 9))
    plt.bar(
        x=badword_index,
        height=top_word_freq_info.loc[badword_index, 'frequency'],
        color='r', label='badwords{goddam, hell, damn, bastard}'
    )
    plt.bar(
        x=non_badword_index,
        height=top_word_freq_info.loc[non_badword_index, 'frequency'],
        color='b'
    )

    plt.legend()
    plt.xticks([])
    plt.xlabel('word')
    plt.ylabel('frequency')
    plt.title('Top Word Frequency Distribution. (feat. badwords)')
    plt.savefig('./output/figures/q2-top_word_frequency.png')

def draw_wordcloud_top_word_frequency_information(top_word_freq_info):
    word_freq = {word: freq for word, freq in zip(
        top_word_freq_info['word'], top_word_freq_info['frequency']
    )}
    wc = WordCloud(
        width=800,
        height=800,
        background_color='white',
        max_words=1000,
        contour_width=3,
        contour_color='firebrick',
        random_state=2020
    )
    wc = wc.generate_from_frequencies(word_freq)

    plt.figure(figsize=(12, 12))
    plt.imshow(wc)

    plt.xticks([])
    plt.yticks([])
    plt.axis('off')
    plt.title('WordCloud for Top Words in Novel')
    plt.savefig('./output/figures/q2-top_word_frequency_wordcloud.png')

if __name__ == "__main__":
    epub_path = \
        './ebook/J. D. Salinger - The Catcher in the Rye '\
        '(1951, Penguin Books Ltd).epub'
    book = load_epub(epub_path)
    book = process_text_data(book)

    word_freq_info = get_word_frequency_information(book)
    top_word_freq_info = get_top_word_frequency_information(
        word_freq_info, topn=200
    )

    badwords = ['goddam', 'hell', 'damn', 'bastard']
    draw_barplot_top_word_frequency_information(top_word_freq_info, badwords)
    draw_wordcloud_top_word_frequency_information(top_word_freq_info)


from django.urls import path
from . import views

urlpatterns =[
    path('',views.login,name='index_login'),
    path('login/',views.login,name='login'),
    path('logout/',views.logout,name='logout'),
    path('cadastrar/',views.cadastrar,name='cadastrar'),
    path('dashboard/',views.dashboard,name='dashboard')
]


# -*- coding: utf8 -*-

from flask_login import current_user
from app import sa
from app.models import Base
from app.helpers import ModelHelper, MutableObject
import datetime

class Comment(Base, sa.Model, ModelHelper):

    __tablename__ = 'comments'

    __json_meta__ = [
        'id',
        'text',
        'profile',
        'post_id',
        'comment_id',
        'children',
        'created_at',
        'modified_at'
    ]

    id = sa.Column(sa.Integer, primary_key=True)
    post_id = sa.Column(sa.Integer,
                        sa.ForeignKey('posts.id',
                                      ondelete='CASCADE',
                                      onupdate='NO ACTION'))
    user_id = sa.Column(sa.Integer,
                        sa.ForeignKey('users.id',
                                      ondelete='CASCADE',
                                      onupdate='NO ACTION'))

    comment_id = sa.Column(sa.Integer, default=0, index=True, nullable=False)
    text = sa.Column(sa.Text)

    created_at = sa.Column(sa.DateTime, default=datetime.datetime.utcnow)
    modified_at = sa.Column(sa.DateTime, default=datetime.datetime.utcnow)

    def __repr__(self):
        return '<Comment %s>' % (self.id)

    @property
    def parent_comment(self):
        if not self.comment_id:
            return None
        if not hasattr(self, '_parent_comment'):
            self._parent_comment = Comment.get_by_id(self.comment_id)
        return self._parent_comment

    @property
    def profile(self):
        if not hasattr(self, '_profile'):
            self._profile = {
                'id': self.user_id,
                'nickname': self.user.nickname,
                'profile_picture_url': self.user.profile_picture_url
            }
        return self._profile

    @property
    def can_edit(self):
        return (current_user.is_authenticated and
                (self.user.id == current_user.id or current_user.is_admin))

    @property
    def can_delete(self):
        return (current_user.is_authenticated and
                (self.user_id == current_user.id or
                 self.post.user_id == current_user.id or
                 current_user.is_admin))

    @property
    def need_reply(self):
        return current_user.is_authenticated and self.user_id != current_user.id

    @property
    def children(self):
        if not hasattr(self, '_children'):
            self._children = []
        return self._children

    @children.setter
    def children(self, value):
        self._children = value


"""
Main function to run the script of the pipeline

Author: Arkaan Quanunga
Date: 03/03/2022

"""
import os
import tempfile
import mlflow
import hydra
import omegaconf

# The Steps this pipeline will run- edit this based on local machine vs S3
# storage
_steps = [
    # Comment this line out if you don't have any data on local machine &
    # already have data present on S3
    # "data_upload_S3",
    "data_upload",
    "data_processing",
    "database_upload"
]

@hydra.main(config_name='config')
def run_pipeline(config: omegaconf.DictConfig):
    """
    Running the entire pipeline flow

    :param config: Getting the dictionary from config file
    :return:
    """

    # Set up the wandb experiment. All runs will be grouped under this name
    os.environ["WANDB_PROJECT"] = config["main"]["project_name"]
    os.environ["WANDB_RUN_GROUP"] = config["main"]["experiment_name"]

    # Steps to execute
    steps_par = config['main']['steps']
    active_steps = steps_par.split(",") if steps_par != "all" else _steps

    # Move to a temporary directory
    with tempfile.TemporaryDirectory() as tmp_dir:

        if "data_upload_S3" in active_steps:
            # Upload files to S3 bucket
            _ = mlflow.run(
                os.path.join(
                    hydra.utils.get_original_cwd(),
                    "src",
                    "components",
                    "data_upload_S3"),
                "main",
                parameters={
                    "AWS_DEFAULT_REGION": config["main"]["AWS"]["DEFAULT_REGION_NAME"],
                    "bucket_name": config["data_upload_S3"]["bucket_name"],
                    "bucket_prefix": config["data_upload_S3"]["bucket_prefix"],
                    "dataset_path": config["data_upload_S3"]["path"],
                })

        if "data_upload" in active_steps:
            # Upload file and load in W&B
            _ = mlflow.run(
                os.path.join(
                    hydra.utils.get_original_cwd(),
                    "src",
                    "components",
                    "data_upload_WandB2"),
                "main",
                parameters={
                    "data_location": config["data_upload_S3"]["data_location"],
                    "bucket_path": "s3://" +
                    config['data_upload_S3']['bucket_name'] +
                    "/" +
                    config["data_upload_S3"]["bucket_prefix"],
                    "directory_path": config["data_upload_S3"]["directory_path"],
                    "output_artifact": "raw_data",
                    "output_type": "data_upload",
                    "output_description": "Artifact for storage of data on Weights & Biases",
                },
            )

        if "data_processing" in active_steps:
            # Process data by extracting crucial files from data uploaded
            _ = mlflow.run(
                os.path.join(
                    hydra.utils.get_original_cwd(),
                    "src",
                    "components",
                    "data_processing"),
                "main",
                parameters={
                    "AWS_DEFAULT_REGION": config["main"]["AWS"]["DEFAULT_REGION_NAME"],
                    "data_upload_type": config["data_upload_S3"]["data_upload_type"],
                    "bucket_name": config["data_upload_S3"]["bucket_name"],
                    "input_artifact": "raw_data:latest",
                    "output_directory": config["data_processing"]["output_directory"],
                    "output_artifact": "processed_data",
                    "output_type": "processed_data",
                    "output_description": "Processing the data by" +
                                          "separating json file to different files &" +
                                          " uploading processed data to S3",
                })

        if "database_upload" in active_steps:
            # Uploading Processed data to database
            _ = mlflow.run(
                os.path.join(
                    os.path.join(
                        hydra.utils.get_original_cwd(),
                        "src",
                        "components",
                        "database_upload")),
                "main",
                parameters={
                    "data_download_type": config["data_processing"]["data_upload_type"],
                    "MONGO_Cluster_name": config["main"]["MONGODB"]["MONGO_Cluster_name"],
                    "Database_name": config["database_upload"]["Database_name"],
                    "input_artifact": "processed_data:latest",
                    "output_artifact": "database_upload",
                    "output_type": "database_upload",
                    "output_description": "Uploading the processed data " +
                                          "to a MONGO DB database for query purposes"})

if __name__ == "__main__":
    run_pipeline()


#!/usr/bin/env python3

import collections

Shader = collections.namedtuple("Shader", "name vs_path fs_path attributes uniforms subroutines")

SHADERS = [
   Shader(
      name = "circle",
      vs_path = "circle.vert",
      fs_path = "circle.frag",
      attributes = ["vertex"],
      uniforms = ["projection", "color", "radius"],
      subroutines = {},
   ),
   Shader(
      name = "circle_filled",
      vs_path = "circle.vert",
      fs_path = "circle_filled.frag",
      attributes = ["vertex"],
      uniforms = ["projection", "color", "radius"],
      subroutines = {},
   ),
   Shader(
      name = "circle_partial",
      vs_path = "circle.vert",
      fs_path = "circle_partial.frag",
      attributes = ["vertex"],
      uniforms = ["projection", "color", "radius", "angle1", "angle2"],
      subroutines = {},
   ),
   Shader(
      name = "solid",
      vs_path = "project.vert",
      fs_path = "solid.frag",
      attributes = ["vertex"],
      uniforms = ["projection", "color"],
      subroutines = {},
   ),
   Shader(
      name = "trail",
      vs_path = "project_pos.vert",
      fs_path = "trail.frag",
      attributes = ["vertex"],
      uniforms = ["projection", "c1", "c2", "t1", "t2", "dt", "pos1", "pos2", "r", "nebu_col" ],
      subroutines = {
        "trail_func" : [
            "trail_default",
            "trail_pulse",
            "trail_wave",
            "trail_flame",
            "trail_nebula",
            "trail_arc",
            "trail_bubbles",
        ]
      }
   ),
   Shader(
      name = "smooth",
      vs_path = "smooth.vert",
      fs_path = "smooth.frag",
      attributes = ["vertex", "vertex_color"],
      uniforms = ["projection"],
      subroutines = {},
   ),
   Shader(
      name = "texture",
      vs_path = "texture.vert",
      fs_path = "texture.frag",
      attributes = ["vertex"],
      uniforms = ["projection", "color", "tex_mat"],
      subroutines = {},
   ),
   Shader(
      name = "texture_interpolate",
      vs_path = "texture.vert",
      fs_path = "texture_interpolate.frag",
      attributes = ["vertex"],
      uniforms = ["projection", "color", "tex_mat", "sampler1", "sampler2", "inter"],
      subroutines = {},
   ),
   Shader(
      name = "nebula",
      vs_path = "nebula.vert",
      fs_path = "nebula_overlay.frag",
      attributes = ["vertex"],
      uniforms = ["projection", "hue", "brightness", "horizon", "eddy_scale", "time"],
      subroutines = {},
   ),
   Shader(
      name = "nebula_background",
      vs_path = "nebula.vert",
      fs_path = "nebula_background.frag",
      attributes = ["vertex"],
      uniforms = ["projection", "hue", "brightness", "eddy_scale", "time"],
      subroutines = {},
   ),
   Shader(
      name = "nebula_map",
      vs_path = "nebula_map.vert",
      fs_path = "nebula_map.frag",
      attributes = ["vertex"],
      uniforms = ["projection", "hue", "eddy_scale", "time", "globalpos", "alpha"],
      subroutines = {},
   ),
   Shader(
      name = "stars",
      vs_path = "stars.vert",
      fs_path = "stars.frag",
      attributes = ["vertex", "brightness"],
      uniforms = ["projection", "star_xy", "wh", "xy", "scale"],
      subroutines = {},
   ),
   Shader(
      name = "font",
      vs_path = "font.vert",
      fs_path = "font.frag",
      attributes = ["vertex", "tex_coord"],
      uniforms = ["projection", "color", "outline_color"],
      subroutines = {},
   ),
   Shader(
      name = "safelanes",
      vs_path = "project_pos.vert",
      fs_path = "safelanes.frag",
      attributes = ["vertex"],
      #uniforms = ["projection", "color", "dt", "r", "dimensions" ],
      uniforms = ["projection", "color", "dimensions" ],
      subroutines = {},
   ),
   Shader(
      name = "beam",
      vs_path = "project_pos.vert",
      fs_path = "beam.frag",
      attributes = ["vertex"],
      uniforms = ["projection", "color", "dt", "r", "dimensions" ],
      subroutines = {
        "beam_func" : [
            "beam_default",
            "beam_wave",
            "beam_arc",
            "beam_helix",
            "beam_organic",
            "beam_unstable",
            "beam_fuzzy",
        ]
      }
   ),
   Shader(
      name = "tk",
      vs_path = "tk.vert",
      fs_path = "tk.frag",
      attributes = ["vertex"],
      uniforms = ["projection", "c", "dc", "lc", "oc", "wh", "corner_radius"],
      subroutines = {},
   ),
   Shader(
      name = "jump",
      vs_path = "project_pos.vert",
      fs_path = "jump.frag",
      attributes = ["vertex"],
      uniforms = ["projection", "progress", "direction", "dimensions"],
      subroutines = {
        "jump_func" : [
            "jump_default",
            "jump_nebula",
            "jump_organic",
            "jump_circular",
            "jump_wind",
        ]
      }
   ),
   Shader(
      name = "colorblind",
      vs_path = "postprocess.vert",
      fs_path = "colorblind.frag",
      attributes = ["VertexPosition"],
      uniforms = ["ClipSpaceFromLocal", "MainTex"],
      subroutines = {},
   ),
   Shader(
      name = "shake",
      vs_path = "postprocess.vert",
      fs_path = "shake.frag",
      attributes = ["VertexPosition"],
      uniforms = ["ClipSpaceFromLocal", "MainTex", "shake_pos", "shake_vel", "shake_force"],
      subroutines = {},
   ),
   Shader(
      name = "damage",
      vs_path = "postprocess.vert",
      fs_path = "damage.frag",
      attributes = ["VertexPosition"],
      uniforms = ["ClipSpaceFromLocal", "MainTex", "damage_strength", "love_ScreenSize"],
      subroutines = {},
   ),
   Shader(
      name = "gamma_correction",
      vs_path = "postprocess.vert",
      fs_path = "gamma_correction.frag",
      attributes = ["VertexPosition"],
      uniforms = ["ClipSpaceFromLocal", "MainTex", "gamma"],
      subroutines = {},
   ),
   Shader(
      name = "status",
      vs_path = "project_pos.vert",
      fs_path = "status.frag",
      attributes = ["vertex"],
      uniforms = ["projection", "ok"],
      subroutines = {},
   ),
]

def write_header(f):
    f.write("/* FILE GENERATED BY %s */\n\n" % __file__)

def generate_h_file(f):
    write_header(f)

    f.write("#ifndef SHADER_GEN_C_H\n")
    f.write("#define SHADER_GEN_C_H\n")

    f.write("#include \"opengl.h\"\n\n")

    f.write("typedef struct Shaders_ {\n")

    for shader in SHADERS:
        f.write("   struct {\n")

        f.write("      GLuint program;\n")

        for attribute in shader.attributes:
            f.write("      GLuint {};\n".format(attribute))

        for uniform in shader.uniforms:
            f.write("      GLuint {};\n".format(uniform))

        for subroutine, routines in shader.subroutines.items():
            f.write("      struct {\n")
            f.write("         GLuint uniform;\n")
            for r in routines:
                f.write(f"         GLuint {r};\n")
            f.write(f"      }} {subroutine};\n")

        f.write("   }} {};\n".format(shader.name))
    f.write("} Shaders;\n\n")

    f.write("extern Shaders shaders;\n\n")

    f.write("void shaders_load (void);\n")
    f.write("void shaders_unload (void);\n")

    f.write("#endif\n")

def generate_c_file(f):
    write_header(f)

    f.write("#include <string.h>\n")
    f.write("#include \"shaders.gen.h\"\n")
    f.write("#include \"opengl_shader.h\"\n\n")

    f.write("Shaders shaders;\n\n")

    f.write("void shaders_load (void) {\n")
    for i, shader in enumerate(SHADERS):
        f.write("   shaders.{}.program = gl_program_vert_frag(\"{}\", \"{}\");\n".format(
                 shader.name,
                 shader.vs_path,
                 shader.fs_path))
        for attribute in shader.attributes:
            f.write("   shaders.{}.{} = glGetAttribLocation(shaders.{}.program, \"{}\");\n".format(
                    shader.name,
                    attribute,
                    shader.name,
                    attribute))

        for uniform in shader.uniforms:
            f.write("   shaders.{}.{} = glGetUniformLocation(shaders.{}.program, \"{}\");\n".format(
                    shader.name,
                    uniform,
                    shader.name,
                    uniform))

        f.write("   if (gl_has( OPENGL_SUBROUTINES )) {\n")
        for subroutine, routines in shader.subroutines.items():
            f.write(f"      shaders.{shader.name}.{subroutine}.uniform = glGetSubroutineUniformLocation( shaders.{shader.name}.program, GL_FRAGMENT_SHADER, \"{subroutine}\" );\n")
            for r in routines:
                f.write(f"      shaders.{shader.name}.{subroutine}.{r} = glGetSubroutineIndex( shaders.{shader.name}.program, GL_FRAGMENT_SHADER, \"{r}\" );\n")
        f.write("   }\n");

        if i != len(SHADERS) - 1:
            f.write("\n")
    f.write("}\n\n")

    f.write("void shaders_unload (void) {\n")
    for shader in SHADERS:
        f.write("   glDeleteProgram(shaders.{}.program);\n".format(shader.name))

    f.write("   memset(&shaders, 0, sizeof(shaders));\n")
    f.write("}\n")

with open("shaders.gen.h", "w") as shaders_gen_h:
    generate_h_file(shaders_gen_h)

with open("shaders.gen.c", "w") as shaders_gen_c:
    generate_c_file(shaders_gen_c)


from SbisSite import SearchHelper
from selenium.webdriver.common.by import By
import time

def test_sbis(browser):
    sbis_main_page = SearchHelper(browser)
# Открываем сайт
    sbis_main_page.go_to_site()
# Проверяем наличие городом из моего региона в списке контактов
    cities_kostroma = ['Кострома', 'Нея']
    for c in cities_kostroma:
        assert c in [x.text for x in sbis_main_page.get_city_names()]
# Открываем список всех регионов, выбираем нужный
    new_region = 'Камчатский край'
    cities_kamchatka = ['Петропавловск-Камчатский']
    sbis_main_page.open_region_chooser()
    region_panel = sbis_main_page.get_region_chooser_panel()
    region_panel.find_element(By.XPATH, "//li/span[contains(@title,'{}')]".format(new_region)).click()
    time.sleep(2)
# Проверяем что url, title и список городов показывают корректную информацию
    assert '41-kamchatskij-kraj' in sbis_main_page.get_current_url()
    for c in cities_kamchatka:
        assert c in [x.text for x in sbis_main_page.get_city_names()]
    assert new_region in sbis_main_page.get_page_title()


import asyncio
import copy
import json
import os
import shlex
import subprocess
import sys
from datetime import datetime
from json.decoder import JSONDecodeError
from typing import Any, Dict, List, cast

import pymongo
import tornado.httpclient
from watchdog import observers
from watchdog.events import FileSystemEvent, FileSystemEventHandler

import kekmonitors.utils.tools
from kekmonitors.comms.msg import Cmd, Response, badResponse, okResponse
from kekmonitors.comms.server import Server
from kekmonitors.config import COMMANDS, ERRORS, Config, LogConfig
from kekmonitors.discord_embeds import get_mm_crash_embed

if sys.version_info[1] > 6:
    import uvloop

def get_parent_directory(src: str) -> str:
    """
    Return the parent directory of `src`
    """
    return src[: src.rfind(os.path.sep)]

class MonitorManager(Server, FileSystemEventHandler):
    """This can be used to manage monitors/scrapers with an external api."""

    def __init__(self, config: Config = None):
        if not config:
            config = Config()
        # set default name if not already set in config
        if not config["OtherConfig"]["socket_name"]:
            config["OtherConfig"]["socket_name"] = f"Executable.MonitorManager"

        self.config = config

        # create logger
        logconfig = LogConfig(config)
        logconfig["OtherConfig"]["socket_name"] += ".General"
        self.general_logger = kekmonitors.utils.tools.get_logger(logconfig)

        if sys.version_info[1] > 6:
            asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
        else:
            self.general_logger.warning(
                f"You're currently running python {sys.version_info[0]}.{sys.version_info[1]}, which does not support uvloop. Please consider upgrading to at least 3.7, since uvloop brings many enhancements to the asyncio loop."
            )

        super().__init__(
            config, f"{self.config['GlobalConfig']['socket_path']}/MonitorManager"
        )  # Server init
        super(Server).__init__()  # FileSystemEventHandler init

        # initialize callbacks
        self.cmd_to_callback[COMMANDS.MM_STOP_MONITOR_MANAGER] = self._stop_serving
        self.cmd_to_callback[COMMANDS.MM_ADD_MONITOR] = self.on_add_monitor
        self.cmd_to_callback[COMMANDS.MM_ADD_SCRAPER] = self.on_add_scraper
        self.cmd_to_callback[
            COMMANDS.MM_ADD_MONITOR_SCRAPER
        ] = self.on_add_monitor_scraper
        self.cmd_to_callback[COMMANDS.MM_STOP_MONITOR] = self.on_stop_monitor
        self.cmd_to_callback[COMMANDS.MM_STOP_SCRAPER] = self.on_stop_scraper
        self.cmd_to_callback[
            COMMANDS.MM_STOP_MONITOR_SCRAPER
        ] = self.on_stop_monitor_scraper
        self.cmd_to_callback[
            COMMANDS.MM_GET_MONITOR_STATUS
        ] = self.on_get_monitor_status
        self.cmd_to_callback[
            COMMANDS.MM_GET_SCRAPER_STATUS
        ] = self.on_get_scraper_status
        self.cmd_to_callback[
            COMMANDS.MM_GET_MONITOR_SCRAPER_STATUS
        ] = self.on_get_status
        self.cmd_to_callback[
            COMMANDS.MM_GET_MONITOR_CONFIG
        ] = self.on_get_monitor_config
        self.cmd_to_callback[
            COMMANDS.MM_SET_MONITOR_CONFIG
        ] = self.on_set_monitor_config
        self.cmd_to_callback[
            COMMANDS.MM_GET_MONITOR_WHITELIST
        ] = self.on_get_monitor_whitelist
        self.cmd_to_callback[
            COMMANDS.MM_SET_MONITOR_WHITELIST
        ] = self.on_set_monitor_whitelist
        self.cmd_to_callback[
            COMMANDS.MM_GET_MONITOR_BLACKLIST
        ] = self.on_get_monitor_blacklist
        self.cmd_to_callback[
            COMMANDS.MM_SET_MONITOR_BLACKLIST
        ] = self.on_set_monitor_blacklist
        self.cmd_to_callback[
            COMMANDS.MM_GET_MONITOR_WEBHOOKS
        ] = self.on_get_monitor_webhooks
        self.cmd_to_callback[
            COMMANDS.MM_SET_MONITOR_WEBHOOKS
        ] = self.on_set_monitor_webhooks
        self.cmd_to_callback[
            COMMANDS.MM_GET_SCRAPER_CONFIG
        ] = self.on_get_scraper_config
        self.cmd_to_callback[
            COMMANDS.MM_SET_SCRAPER_CONFIG
        ] = self.on_set_scraper_config
        self.cmd_to_callback[
            COMMANDS.MM_GET_SCRAPER_WHITELIST
        ] = self.on_get_scraper_whitelist
        self.cmd_to_callback[
            COMMANDS.MM_SET_SCRAPER_WHITELIST
        ] = self.on_set_scraper_whitelist
        self.cmd_to_callback[
            COMMANDS.MM_GET_SCRAPER_BLACKLIST
        ] = self.on_get_scraper_blacklist
        self.cmd_to_callback[
            COMMANDS.MM_SET_SCRAPER_BLACKLIST
        ] = self.on_set_scraper_blacklist
        self.cmd_to_callback[
            COMMANDS.MM_GET_SCRAPER_WEBHOOKS
        ] = self.on_get_scraper_webhooks
        self.cmd_to_callback[
            COMMANDS.MM_SET_SCRAPER_WEBHOOKS
        ] = self.on_set_scraper_webhooks
        self.cmd_to_callback[
            COMMANDS.MM_SET_MONITOR_SCRAPER_BLACKLIST
        ] = self.on_set_monitor_scraper_blacklist
        self.cmd_to_callback[
            COMMANDS.MM_SET_MONITOR_SCRAPER_WHITELIST
        ] = self.on_set_monitor_scraper_whitelist
        self.cmd_to_callback[
            COMMANDS.MM_SET_MONITOR_SCRAPER_WEBHOOKS
        ] = self.on_set_monitor_scraper_webhooks
        self.cmd_to_callback[
            COMMANDS.MM_SET_MONITOR_SCRAPER_CONFIG
        ] = self.on_set_monitor_scraper_configs
        self.cmd_to_callback[COMMANDS.MM_GET_SCRAPER_SHOES] = self.on_get_scraper_shoes
        self.cmd_to_callback[COMMANDS.MM_GET_MONITOR_SHOES] = self.on_get_monitor_shoes

        # initialize variables
        self.monitor_processes = {}  # type: Dict[str, Dict[str, Any]]
        self.scraper_processes = {}  # type: Dict[str, Dict[str, Any]]
        self.monitor_sockets = {}  # type: Dict[str, str]
        self.scraper_sockets = {}  # type: Dict[str, str]
        self.register_db = pymongo.MongoClient(
            self.config["GlobalConfig"]["db_path"]
        )[  # database where to find class_name -> filename relation
            self.config["GlobalConfig"]["db_name"]
        ][
            "register"
        ]

        # let's avoid concurrency problems on socket creation/deletion
        self.socket_lock = asyncio.Lock()
        # and don't stop on MM_STOP_MONITOR_MANAGER if we are in the loop
        self._loop_lock = asyncio.Lock()

        # mandatory arguments, needed in the command
        self.add_args = ["name"]
        self.stop_args = ["name"]
        self.getter_args = ["name"]
        self.setter_args = ["name", "payload"]

        self.shutdown_all_on_exit = True  # you might wanna change this

        # needed for the crash webhook
        tornado.httpclient.AsyncHTTPClient.configure(
            "tornado.curl_httpclient.CurlAsyncHTTPClient"
        )
        self.client = tornado.httpclient.AsyncHTTPClient()

        # create main loop task
        self.check_status_task = self._asyncio_loop.create_task(self.check_status())

        # watches the config folder for any change. calls on_modified when any monitored file is modified
        self.watcher = observers.Observer()
        self.watcher.schedule(self, self.config["GlobalConfig"]["config_path"], True)
        self.watcher.schedule(self, self.config["GlobalConfig"]["socket_path"], True)

        # needed for proper shutdown
        self.has_to_quit = False

    def start(self):
        """Start the Monitor Manager."""
        self.watcher.start()
        self._asyncio_loop.run_forever()

    def on_modified(self, event: FileSystemEvent):
        # called when any of the monitored files is modified.
        # we are only interested int the configs for now.
        filepath = event.key[1]  # type: str

        if not filepath.endswith(".json"):
            return
        # if a config file is updated:
        if filepath.startswith(self.config["GlobalConfig"]["config_path"]):
            if filepath.startswith(
                os.path.sep.join((self.config["GlobalConfig"]["config_path"], "common"))
            ):
                asyncio.run_coroutine_threadsafe(
                    self.update_common_config(filepath), self._asyncio_loop
                )
            else:
                asyncio.run_coroutine_threadsafe(
                    self.update_specific_config(filepath), self._asyncio_loop
                )

    def on_created(self, event: FileSystemEvent):
        filename = event.key[1]  # type: str

        # if a socket was created:
        if filename.find(self.config["GlobalConfig"]["socket_path"]) != -1:
            asyncio.run_coroutine_threadsafe(self.on_add_sockets(), self._asyncio_loop)

    def on_deleted(self, event):
        filename = event.key[1]  # type: str

        # if a socket was deleted:
        if filename.find(self.config["GlobalConfig"]["socket_path"]) != -1:
            asyncio.run_coroutine_threadsafe(
                self.on_delete_sockets(), self._asyncio_loop
            )

    async def on_add_sockets(self):
        """
        Routine that updates the internal list of available sockets on add event
        """
        async with self.socket_lock:
            # get any socket that is not in the list
            new_monitor_sockets = {}  # type: Dict[str, str]
            new_scraper_sockets = {}  # type: Dict[str, str]
            for filename in os.listdir(self.config["GlobalConfig"]["socket_path"]):
                splits = filename.split(".")
                if splits[0] == "Monitor" and splits[1] not in self.monitor_sockets:
                    new_monitor_sockets[splits[1]] = os.path.sep.join(
                        [self.config["GlobalConfig"]["socket_path"], filename]
                    )
                elif splits[0] == "Scraper" and splits[1] not in self.scraper_sockets:
                    new_scraper_sockets[splits[1]] = os.path.sep.join(
                        [self.config["GlobalConfig"]["socket_path"], filename]
                    )

            # check if it's alive
            alive_monitor_sockets, alive_scraper_sockets = await asyncio.gather(
                self.get_alive_sockets(new_monitor_sockets.values()),
                self.get_alive_sockets(new_scraper_sockets.values()),
            )

            # if it is, add it to the list
            for class_name in new_monitor_sockets:
                if new_monitor_sockets[class_name] in alive_monitor_sockets:
                    self.monitor_sockets[class_name] = new_monitor_sockets[class_name]

            for class_name in new_scraper_sockets:
                if new_scraper_sockets[class_name] in alive_scraper_sockets:
                    self.scraper_sockets[class_name] = new_scraper_sockets[class_name]

    async def on_delete_sockets(self):
        """
        Routine that updates the internal list of available sockets on delete event
        """
        async with self.socket_lock:
            existing_monitor_sockets = []
            existing_scraper_sockets = []
            # get the current list of existing sockets
            for f in list(os.listdir(self.config["GlobalConfig"]["socket_path"])):
                if f.startswith("Monitor."):
                    existing_monitor_sockets.append(f)
                elif f.startswith("Scraper."):
                    existing_scraper_sockets.append(f)

            # temp copy
            updated_monitor_sockets = copy.copy(self.monitor_sockets)
            updated_scraper_sockets = copy.copy(self.scraper_sockets)

            # remove every internal socket that is not existing
            for class_name in self.monitor_sockets:
                if "Monitor." + class_name not in existing_monitor_sockets:
                    updated_monitor_sockets.pop(class_name)
            for class_name in self.scraper_sockets:
                if "Scraper." + class_name not in existing_scraper_sockets:
                    updated_scraper_sockets.pop(class_name)

            self.monitor_sockets = updated_monitor_sockets
            self.scraper_sockets = updated_scraper_sockets

    async def get_alive_sockets(self, sockets: List[str]) -> List[str]:
        """
        Ping the provided sockets and return a list of alive sockets
        """
        tasks = []
        for socket in sockets:
            cmd = Cmd()
            cmd.cmd = COMMANDS.PING
            tasks.append(self.make_request(socket, cmd))

        responses = await asyncio.gather(*tasks)  # type: List[Response]
        alive = []
        for response, socket in zip(responses, sockets):
            if not response.error.value:
                alive.append(socket)

        return alive

    async def update_common_config(self, filename: str):
        """Reads the provided config file and updates the interested monitors/scrapers"""
        self.general_logger.debug(f"File {filename} has changed!")
        try:
            with open(filename, "r") as f:
                j = json.load(f)
        except JSONDecodeError:
            self.general_logger.warning(
                f"File {filename} has changed but contains invalid json data"
            )
            return

        splits = filename.split(os.path.sep)

        for sockets in (self.monitor_sockets, self.scraper_sockets):
            commands = []  # List[Cmd]
            sock_paths = []  # type: List[str]

            # we are interested in configs, whitelists, blacklists, webhooks
            if splits[-1] == "whitelists.json":
                cmd = COMMANDS.SET_COMMON_WHITELIST
            elif splits[-1] == "configs.json":
                cmd = COMMANDS.SET_COMMON_CONFIG
            elif splits[-1] == "blacklists.json":
                cmd = COMMANDS.SET_COMMON_BLACKLIST
            elif splits[-1] == "webhooks.json":
                cmd = COMMANDS.SET_COMMON_WEBHOOKS
            else:
                return

            # for every monitor socket
            for name in sockets:
                if name in j:
                    sock_path = sockets[name]
                    c = Cmd()
                    c.cmd = cmd
                    # send only the corresponding part to the monitor
                    c.payload = j[name]
                    commands.append(c)
                    sock_paths.append(sock_path)

            # prepare to make all the async requests
            tasks = []
            for sock_path, command in zip(sock_paths, commands):
                tasks.append(self.make_request(sock_path, command))

            # send the requests
            responses = await asyncio.gather(*tasks)  # List[Response]

            for response in responses:
                if response.error.value:
                    self.general_logger.warning(
                        f"Failed to update config: {response.error}"
                    )

    async def update_specific_config(self, filename: str):
        """Reads the provided config file and updates the interested monitors/scrapers"""
        self.general_logger.debug(f"File {filename} has changed!")
        try:
            with open(filename, "r") as f:
                j = json.load(f)
        except JSONDecodeError:
            self.general_logger.warning(
                f"File {filename} has changed but contains invalid json data"
            )
            return

        splits = filename.split(os.path.sep)
        commands = []  # List[Cmd]
        sock_paths = []  # type: List[str]

        # if it's from the monitors folder:
        if "monitors" in filename.split(os.path.sep):
            sockets = self.monitor_sockets
        elif "scrapers" in filename.split(os.path.sep):
            sockets = self.scraper_sockets
        else:
            self.general_logger.debug("File not useful.")
            return

        # we are interested in configs, whitelists, blacklists, webhooks
        if splits[-1] == "whitelists.json":
            cmd = COMMANDS.SET_SPECIFIC_WHITELIST
        elif splits[-1] == "configs.json":
            cmd = COMMANDS.SET_SPECIFIC_CONFIG
        elif splits[-1] == "blacklists.json":
            cmd = COMMANDS.SET_SPECIFIC_BLACKLIST
        elif splits[-1] == "webhooks.json":
            cmd = COMMANDS.SET_SPECIFIC_WEBHOOKS
        else:
            return

        # for every monitor socket
        for name in sockets:
            if name in j:
                sock_path = sockets[name]
                c = Cmd()
                c.cmd = cmd
                # send only the corresponding part to the monitor
                c.payload = j[name]
                commands.append(c)
                sock_paths.append(sock_path)

        # prepare to make all the async requests
        tasks = []
        for sock_path, command in zip(sock_paths, commands):
            tasks.append(self.make_request(sock_path, command))

        # send the requests
        responses = await asyncio.gather(*tasks)  # List[Response]

        for response in responses:
            if response.error.value:
                self.general_logger.warning(
                    f"Failed to update config: {response.error}"
                )

    async def on_server_stop(self):
        """
        Routine that runs on server stop. Shuts down the monitor manager
        """
        async with self._loop_lock:
            # stop the config watcher
            self.watcher.stop()
            self.watcher.join()

            if self.shutdown_all_on_exit:
                # get all the existing sockets
                sockets = []  # type: List[str]
                tasks = []
                for sockname in os.listdir(self.config["GlobalConfig"]["socket_path"]):
                    if sockname.startswith("Scraper.") or sockname.startswith(
                        "Monitor."
                    ):
                        cmd = Cmd()
                        cmd.cmd = COMMANDS.STOP
                        sockets.append(sockname)
                        self.general_logger.info(f"Stopping {sockname}...")
                        tasks.append(
                            self.make_request(
                                f"{self.config['GlobalConfig']['socket_path']}{os.path.sep}{sockname}",
                                cmd,
                            )
                        )

                # send request to stop
                responses = await asyncio.gather(*tasks)  # type: List[Response]

                for sockname, r in zip(sockets, responses):
                    # if an error happened...
                    if r.error.value:
                        # if the socket was not used remove it
                        if r.error == ERRORS.SOCKET_COULDNT_CONNECT:
                            os.remove(
                                os.path.sep.join(
                                    [
                                        self.config["GlobalConfig"]["socket_path"],
                                        sockname,
                                    ]
                                )
                            )
                            self.general_logger.info(
                                f"{self.config['GlobalConfig']['socket_path']}{os.path.sep}{sockname} was removed because unavailable"
                            )
                        # else something else happened, dont do anything
                        else:
                            self.general_logger.warning(
                                f"Error occurred while attempting to stop {sockname}: {r.error}"
                            )
                    # ok
                    else:
                        self.general_logger.info(f"{sockname} was successfully stopped")

        self._asyncio_loop.stop()
        self.general_logger.info("Shutting down...")
        return okResponse()

    async def check_status(self):
        """Main MonitorManager loop. Every second it checks its monitored processes and looks if they are still alive, possibly reporting any exit code"""
        while True:
            async with self._loop_lock:
                new_monitor_processes = {}
                for class_name in self.monitor_processes:
                    monitor = self.monitor_processes[class_name]["process"]
                    if monitor.poll() is not None:
                        log = f"Monitor {class_name} has stopped with code: {monitor.returncode}"
                        if monitor.returncode:
                            self.general_logger.warning(log)
                            if self.config["WebhookConfig"]["crash_webhook"]:
                                embed = get_mm_crash_embed(
                                    "Monitor " + class_name,
                                    monitor.returncode,
                                    monitor.pid,
                                )
                                ts = datetime.now().strftime(
                                    self.config["WebhookConfig"]["timestamp_format"]
                                )

                                embed.set_footer(
                                    text=f"{self.config['WebhookConfig']['provider']} | {ts}",
                                    icon_url=self.config["WebhookConfig"][
                                        "provider_icon"
                                    ],
                                )
                                data = json.dumps(
                                    {
                                        "embeds": [embed.to_dict()],
                                        "username": "MonitorManager process watcher",
                                        "avatar_url": self.config["WebhookConfig"][
                                            "provider_icon"
                                        ],
                                    }
                                )
                                r = await self.client.fetch(
                                    self.config["WebhookConfig"]["crash_webhook"],
                                    method="POST",
                                    body=data,
                                    headers={"content-type": "application/json"},
                                    raise_error=False,
                                )
                        else:
                            self.general_logger.info(log)
                    else:
                        new_monitor_processes[class_name] = self.monitor_processes[
                            class_name
                        ]
                self.monitor_processes = new_monitor_processes

                new_scraper_processes = {}
                for class_name in self.scraper_processes:
                    scraper = self.scraper_processes[class_name]["process"]
                    if scraper.poll() is not None:
                        log = f"Scraper {class_name} has stopped with code: {scraper.returncode}"
                        if scraper.returncode:
                            self.general_logger.warning(log)
                            if self.config["WebhookConfig"]["crash_webhook"]:
                                embed = get_mm_crash_embed(
                                    "Scraper " + class_name,
                                    scraper.returncode,
                                    scraper.pid,
                                )
                                ts = datetime.now().strftime(
                                    self.config["WebhookConfig"]["timestamp_format"]
                                )

                                embed.set_footer(
                                    text=f"{self.config['WebhookConfig']['provider']} | {ts}",
                                    icon_url=self.config["WebhookConfig"][
                                        "provider_icon"
                                    ],
                                )
                                data = json.dumps(
                                    {
                                        "embeds": [embed.to_dict()],
                                        "username": "MonitorManager process watcher",
                                        "avatar_url": self.config["WebhookConfig"][
                                            "provider_icon"
                                        ],
                                    }
                                )
                                r = await self.client.fetch(
                                    self.config["WebhookConfig"]["crash_webhook"],
                                    method="POST",
                                    body=data,
                                    headers={"content-type": "application/json"},
                                    raise_error=False,
                                )
                        else:
                            self.general_logger.info(log)
                    else:
                        new_scraper_processes[class_name] = self.scraper_processes[
                            class_name
                        ]
                self.scraper_processes = new_scraper_processes
            await asyncio.sleep(1)

    async def on_add_monitor(self, cmd: Cmd) -> Response:
        r = badResponse()
        success, missing = cmd.has_valid_args(self.add_args)
        if success:
            payload = cast(Dict[str, Any], cmd.payload)
            db_monitor = self.register_db["monitors"].find_one(
                {"name": payload["name"]}
            )
            if db_monitor:
                success, reason = await self.add_monitor(db_monitor["path"], payload)
                if success:
                    r = okResponse()
                else:
                    r.error = ERRORS.MM_COULDNT_ADD_MONITOR
                    r.info = reason
            else:
                r.error = ERRORS.MONITOR_NOT_REGISTERED
                r.info = f"Tried to add monitor {payload['name']} but it was not found in the db. Did you start it at least once manually?"
        else:
            r.error = ERRORS.MISSING_PAYLOAD_ARGS
            r.info = f"Missing arguments: {missing}"
        return r

    async def on_add_scraper(self, cmd: Cmd) -> Response:
        r = badResponse()
        success, missing = cmd.has_valid_args(self.add_args)
        if success:
            payload = cast(Dict[str, Any], cmd.payload)
            db_scraper = self.register_db["scrapers"].find_one(
                {"name": payload["name"]}
            )
            if db_scraper:
                success, reason = await self.add_scraper(db_scraper["path"], payload)
                if success:
                    r = okResponse()
                else:
                    r.error = ERRORS.MM_COULDNT_ADD_SCRAPER
                    r.info = reason
            else:
                r.error = ERRORS.SCRAPER_NOT_REGISTERED
                r.info = f"Tried to add scraper {payload['name']} but it was not found in the db. Did you start it at least once manually?"
        else:
            r.error = ERRORS.MISSING_PAYLOAD_ARGS
            r.info = f"Missing arguments: {missing}"

        return r

    async def on_add_monitor_scraper(self, cmd: Cmd) -> Response:
        r = Response()
        cmd1 = cmd
        cmd2 = copy.copy(cmd)
        cmd2.payload = copy.deepcopy(cmd1.payload)
        r1, r2 = await asyncio.gather(
            self.on_add_monitor(cmd1), self.on_add_scraper(cmd2)
        )
        r.error = (
            ERRORS.OK
            if not r1.error.value and not r2.error.value
            else ERRORS.MM_COULDNT_ADD_MONITOR_SCRAPER
        )
        r.info = f"Monitor: {r1.error.name}, Scraper: {r2.error.name}"
        if r.error.value and r.error:
            self.general_logger.warning(f"Couldn't add monitor and scraper")
            kekmonitors.utils.tools.dump_error(self.general_logger, r)

        return r

    async def on_stop_monitor(self, cmd: Cmd) -> Response:
        r = badResponse()
        success, missing = cmd.has_valid_args(self.stop_args)
        if success:
            payload = cast(Dict[str, Any], cmd.payload)
            socket = f"{self.config['GlobalConfig']['socket_path']}/Monitor.{payload['name']}"
            command = Cmd()
            command.cmd = COMMANDS.STOP
            self.general_logger.debug(f"Sending STOP to {socket}...")
            r = await self.make_request(socket, command)
            self.general_logger.debug(f"Sent STOP to {socket}")
        else:
            r.error = ERRORS.MISSING_PAYLOAD_ARGS
            r.info = f"Missing arguments: {missing}"
        return r

    async def on_stop_scraper(self, cmd: Cmd) -> Response:
        r = badResponse()
        success, missing = cmd.has_valid_args(self.stop_args)
        if success:
            payload = cast(Dict[str, Any], cmd.payload)
            socket = f"{self.config['GlobalConfig']['socket_path']}/Scraper.{payload['name']}"
            command = Cmd()
            command.cmd = COMMANDS.STOP
            self.general_logger.debug(f"Sending STOP to {socket}...")
            r = await self.make_request(socket, command)
            self.general_logger.debug(f"Sent STOP to {socket}")
        else:
            r.error = ERRORS.MISSING_PAYLOAD_ARGS
            r.info = f"Missing arguments: {missing}"
        return r

    async def on_stop_monitor_scraper(self, cmd: Cmd) -> Response:
        r = badResponse()
        success, missing = cmd.has_valid_args(self.stop_args)
        if success:
            r1, r2 = await asyncio.gather(
                self.on_stop_monitor(cmd), self.on_stop_scraper(cmd)
            )
            r.error = (
                ERRORS.OK
                if not r1.error.value and not r2.error.value
                else ERRORS.MM_COULDNT_STOP_MONITOR_SCRAPER
            )
            r.info = f"Monitor: {r1.error.name}, Scraper: {r2.error.name}"
            if r.error.value and r.error:
                self.general_logger.warning(f"Couldn't stop monitor and scraper")
                kekmonitors.utils.tools.dump_error(self.general_logger, r)
        else:
            r.error = ERRORS.MISSING_PAYLOAD_ARGS
            r.info = f"Missing arguments: {missing}"
        return r

    async def on_get_monitor_status(self, cmd: Cmd) -> Response:
        process_status = {}
        for class_name in self.monitor_processes:
            start = self.monitor_processes[class_name]["start"].strftime(
                "%m/%d/%Y, %H:%M:%S"
            )
            process_status[class_name] = {
                "Started at": start,
                "PID": self.monitor_processes[class_name]["process"].pid,
            }
        sockets_status = {}
        # for class_name in self.monitor_sockets:
        # sockets_status[class_name] = {class_name: self.monitor_sockets[class_name]}
        sockets_status = self.monitor_sockets
        response = okResponse()
        response.payload = {
            "monitored_processes": process_status,
            "available_sockets": sockets_status,
        }
        return response

    async def on_get_scraper_status(self, cmd: Cmd) -> Response:
        process_status = {}
        for class_name in self.scraper_processes:
            start = self.scraper_processes[class_name]["start"].strftime(
                "%m/%d/%Y, %H:%M:%S"
            )
            process_status[class_name] = {
                "Started at": start,
                "PID": self.scraper_processes[class_name]["process"].pid,
            }
        sockets_status = {}
        # for class_name in self.scraper_sockets:
        # sockets_status[class_name] = {class_name: self.scraper_sockets[class_name]}
        sockets_status = self.scraper_sockets
        response = okResponse()
        response.payload = {
            "monitored_processes": process_status,
            "available_sockets": sockets_status,
        }
        return response

    async def on_get_status(self, cmd: Cmd) -> Response:
        ms = await self.on_get_monitor_status(cmd)
        ss = await self.on_get_scraper_status(cmd)
        response = okResponse()
        msp = cast(Dict[str, Any], ms.payload)  # type: Dict[str, Any]
        ssp = cast(Dict[str, Any], ss.payload)  # type: Dict[str, Any]
        response.payload = {"monitors": msp, "scrapers": ssp}
        return response

    async def add_monitor(self, filename: str, kwargs: Dict[str, str]):
        class_name = kwargs.pop("name")

        if class_name in self.monitor_processes:
            self.general_logger.debug(
                f"Tried to add an already existing monitor ({class_name} ({filename}))"
            )
            return False, "Monitor already started."

        first_part_cmd = (
            f"nohup {sys.executable} {filename} --no-output --no-config-watcher"
        )

        args = []
        for key in kwargs:
            args.append(f"--{key} {kwargs[key]}")

        cmd = " ".join((first_part_cmd, *args))

        self.general_logger.debug(f"Starting {class_name} ({filename})...")
        monitor = subprocess.Popen(
            shlex.split(cmd), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
        )

        await asyncio.sleep(2)  # wait to check if process is still alive

        if monitor.poll() is not None:
            success = False
            msg = f"Failed to start monitor {class_name} ({filename})"
            self.general_logger.warning(
                f"Tried to start {class_name} ({filename}) but failed"
            )
        else:
            self.general_logger.info(
                f"Added monitor {class_name} with pid {monitor.pid}"
            )
            self.monitor_processes[class_name] = {
                "process": monitor,
                "start": datetime.now(),
            }
            success = True
            msg = ""
        return success, msg

    async def add_scraper(self, filename: str, kwargs: Dict[str, str]):
        class_name = kwargs.pop("name")

        if class_name in self.scraper_processes:
            self.general_logger.debug(
                f"Tried to add an already existing scraper ({class_name} ({filename}))"
            )
            return False, "Scraper already started."

        first_part_cmd = (
            f"nohup {sys.executable} {filename} --no-output --no-config-watcher"
        )

        args = []
        for key in kwargs:
            args.append(f"--{key} {kwargs[key]}")

        cmd = " ".join((first_part_cmd, " ".join(args)))

        self.general_logger.debug(f"Starting {class_name} ({filename})...")
        scraper = subprocess.Popen(
            shlex.split(cmd), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
        )

        await asyncio.sleep(2)  # wait to check if process is still alive

        if scraper.poll() is not None:
            success = False
            msg = f"Failed to start scraper {class_name} ({filename})"
            self.general_logger.warning(
                f"Tried to start {class_name} ({filename}) but failed"
            )
        else:
            self.general_logger.info(
                f"Added scraper {class_name} with pid {scraper.pid}"
            )
            self.scraper_processes[class_name] = {
                "process": scraper,
                "start": datetime.now(),
            }
            success = True
            msg = ""
        return success, msg

    async def on_get_monitor_config(self, cmd: Cmd) -> Response:
        return await self.specific_config_getter(cmd, COMMANDS.GET_CONFIG, True)

    async def on_set_monitor_config(self, cmd: Cmd) -> Response:
        return await self.specific_config_setter(cmd, "configs.json", True)

    async def on_get_monitor_whitelist(self, cmd: Cmd) -> Response:
        return await self.specific_config_getter(cmd, COMMANDS.GET_WHITELIST, True)

    async def on_set_monitor_whitelist(self, cmd: Cmd) -> Response:
        return await self.specific_config_setter(cmd, "whitelists.json", True)

    async def on_get_monitor_blacklist(self, cmd: Cmd) -> Response:
        return await self.specific_config_getter(cmd, COMMANDS.GET_BLACKLIST, True)

    async def on_set_monitor_blacklist(self, cmd: Cmd) -> Response:
        return await self.specific_config_setter(cmd, "blacklists.json", True)

    async def on_get_monitor_webhooks(self, cmd: Cmd) -> Response:
        return await self.specific_config_getter(cmd, COMMANDS.GET_WEBHOOKS, True)

    async def on_set_monitor_webhooks(self, cmd: Cmd) -> Response:
        return await self.specific_config_setter(cmd, "webhooks.json", True)

    async def on_get_scraper_config(self, cmd: Cmd) -> Response:
        return await self.specific_config_getter(cmd, COMMANDS.GET_CONFIG, False)

    async def on_set_scraper_config(self, cmd: Cmd) -> Response:
        return await self.specific_config_setter(cmd, "configs.json", False)

    async def on_get_scraper_whitelist(self, cmd: Cmd) -> Response:
        return await self.specific_config_getter(cmd, COMMANDS.GET_WHITELIST, False)

    async def on_set_scraper_whitelist(self, cmd: Cmd) -> Response:
        return await self.specific_config_setter(cmd, "whitelists.json", False)

    async def on_get_scraper_blacklist(self, cmd: Cmd) -> Response:
        return await self.specific_config_getter(cmd, COMMANDS.GET_BLACKLIST, False)

    async def on_set_scraper_blacklist(self, cmd: Cmd) -> Response:
        return await self.specific_config_setter(cmd, "blacklists.json", False)

    async def on_get_scraper_webhooks(self, cmd: Cmd) -> Response:
        return await self.specific_config_getter(cmd, COMMANDS.GET_WEBHOOKS, False)

    async def on_set_scraper_webhooks(self, cmd: Cmd) -> Response:
        return await self.specific_config_setter(cmd, "webhooks.json", False)

    async def on_set_monitor_scraper_blacklist(self, cmd: Cmd) -> Response:
        return await self.common_config_setter(cmd, "blacklists.json")

    async def on_set_monitor_scraper_whitelist(self, cmd: Cmd) -> Response:
        return await self.common_config_setter(cmd, "whitelists.json")

    async def on_set_monitor_scraper_webhooks(self, cmd: Cmd) -> Response:
        return await self.common_config_setter(cmd, "webhooks.json")

    async def on_set_monitor_scraper_configs(self, cmd: Cmd) -> Response:
        return await self.common_config_setter(cmd, "configs.json")

    async def specific_config_getter(
        self, cmd: Cmd, command: COMMANDS, is_monitor: bool
    ):
        success, missing = cmd.has_valid_args(self.getter_args)
        if success:
            payload = cast(Dict[str, Any], cmd.payload)
            c = Cmd()
            c.cmd = command
            r = await self.make_request(
                f"{self.config['GlobalConfig']['socket_path']}/{'Monitor' if is_monitor else 'Scraper'}.{payload['name']}",
                c,
            )
            return r
        else:
            r = badResponse()
            r.error = ERRORS.MISSING_PAYLOAD_ARGS
            r.info = f"{missing}"
            return r

    async def specific_config_setter(self, cmd: Cmd, filename: str, is_monitor: bool):
        success, missing = cmd.has_valid_args(self.setter_args)
        if success:
            payload = cast(Dict[str, Any], cmd.payload)
            cp = os.path.sep.join(
                (
                    self.config["GlobalConfig"]["config_path"],
                    "monitors" if is_monitor else "scrapers",
                    filename,
                )
            )
            with open(
                cp,
                "r",
            ) as rf:
                f = json.load(rf)
            f[payload["name"]] = payload["payload"]
            with open(
                cp,
                "w",
            ) as wf:
                json.dump(f, wf)
        else:
            r = badResponse()
            r.error = ERRORS.MISSING_PAYLOAD_ARGS
            r.info = f"{missing}"
            return r

    async def common_config_setter(self, cmd: Cmd, filename: str):
        success, missing = cmd.has_valid_args(self.setter_args)
        if success:
            payload = cast(Dict[str, Any], cmd.payload)
            cp = os.path.sep.join(
                (self.config["GlobalConfig"]["config_path"], "common", filename)
            )
            with open(
                cp,
                "r",
            ) as rf:
                f = json.load(rf)
            f[payload["name"]] = payload["payload"]
            with open(
                cp,
                "w",
            ) as wf:
                json.dump(f, wf)
        else:
            r = badResponse()
            r.error = ERRORS.MISSING_PAYLOAD_ARGS
            r.info = f"{missing}"
            return r

    async def on_get_scraper_shoes(self, cmd: Cmd) -> Response:
        success, missing = cmd.has_valid_args(self.getter_args)
        if success:
            payload = cast(Dict[str, Any], cmd.payload)
            c = Cmd()
            c.cmd = COMMANDS.GET_SHOES
            r = await self.make_request(
                f"{self.config['GlobalConfig']['socket_path']}/Monitor.{payload['name']}",
                c,
            )
            return r
        else:
            r = badResponse()
            r.error = ERRORS.MISSING_PAYLOAD_ARGS
            r.info = f"{missing}"
            return r

    async def on_get_monitor_shoes(self, cmd: Cmd) -> Response:
        success, missing = cmd.has_valid_args(self.getter_args)
        if success:
            payload = cast(Dict[str, Any], cmd.payload)
            c = Cmd()
            c.cmd = COMMANDS.SET_SHOES
            r = await self.make_request(
                f"{self.config['GlobalConfig']['socket_path']}/Monitor.{payload['name']}",
                c,
            )
            return r
        else:
            r = badResponse()
            r.error = ERRORS.MISSING_PAYLOAD_ARGS
            r.info = f"{missing}"
            return r

if __name__ == "__main__":
    MonitorManager().start()


from app import db
from app.models import \
    MoneroTransactions, MoneroWalletFee
from app.generalfunctions import floating_decimals
from sqlalchemy import func

def getlastestfee():

    """
    THis will query the last 10 withdrawls...get the average fee
    :return:
    """

    getratings = db.session.query(func.avg(MoneroTransactions.fee))
    getratings = getratings.filter(MoneroTransactions.category == 2)
    getratings = getratings.order_by(MoneroTransactions.created.desc())
    avgrate = getratings.limit(10)
    x = (avgrate[0][0])
    y = floating_decimals(x, 8)

    thefee = db.session.query(MoneroWalletFee).filter_by(id=1).first()
    thefee.amount = y
    db.session.add(thefee)
    db.session.commit()
getlastestfee()


import pymongo
from pymongo import MongoClient

cluster = MongoClient("mongodb://localhost:27017")
database = cluster["Storage"]
collection = database["Inventory"]

deletingManyDate = collection.delete_many({})
post = {"_id": 0,
        "quantity": 400,
        "productId": 0,
        "productName": "Soap",
        "price": 200,
        "category": "Personal Protective Equipment (PPE)",
        "vendor": "Vendor1",
        "added_count": 20}

post1 = {"_id": 1,
         "quantity": 400,
         "productId": 1,
         "productName": "Brush",
         "price": 400,
         "category": "Surgical Supplies",
         "vendor": "Vendor2",
         "added_count": 10}

post2 = {"_id": 2,
         "quantity": 800,
         "productId": 2,
         "productName": "Mask",
         "price": 10,
         "category": "Laboratory Supplies",
         "vendor": "Vendor1",
         "added_count": 15}

post3 = {"_id": 3,
         "quantity": 100,
         "productId": 3,
         "productName": "Sanitizers",
         "price": 300,
         "category": "Sterile instruments",
         "vendor": "Vendor2",
         "added_count": 2}

post4 = {"_id": 4,
         "quantity": 100,
         "productId": 4,
         "productName": "Sanitizers",
         "price": 300,
         "category": "Pharmaceuticals",
         "vendor": "Vendor1",
         "added_count": 1}

post5 = {"_id": 5,
         "quantity": 50,
         "productId": 5,
         "productName": "Condoms",
         "price": 2.99,
         "category": "Diagnostic Equipment",
         "vendor": "Vendor2",
         "added_count": 6}

post6 = {"_id": 6,
         "quantity": 20,
         "productId": 6,
         "productName": "Medicines",
         "price": 10,
         "category": "Wound Care Supplies",
         "vendor": "Vendor1",
         "added_count": 7}

post7 = {"_id": 7,
         "quantity": 20,
         "productId": 7,
         "productName": "Mac",
         "price": 20,
         "category": "Laboratory Supplies",
         "vendor": "Vendor1",
         "added_count": 8}

# Inserting multiple lines
collection.insert_many([post1, post2, post3, post4, post5, post6, post7])

sorted_cursor = collection.find().sort('productName', 1)

sorted_docs = []

for doc in sorted_cursor:
    sorted_docs.append(doc)

collection.delete_many({})

collection.insert_many(sorted_docs)