from django.urls import path
from . import views

urlpatterns = [
    path('', views.home, name='home'),
    path('forms', views.form, name='form'),
    path('remove/<todo_id>', views.delete_todo, name='remove'),
    path('add_todo', views.add_todo, name='new_todo'),
    path('update_todo/<todo_id>', views.edit_todo, name='edit_todo'),
    path('chart', views.chart_view, name='chart-view'),
    path('products', views.add_product, name='add-product'),
    path('management', views.add_managers, name='managers'),
    path('depart', views.add_department, name='department'),

]


from flask import Flask, request, jsonify, url_for, session, redirect, render_template
from flaskext.mysql import MySQL
from flask_oauth import OAuth
import logging
import time
from logging.handlers import RotatingFileHandler
# from urllib.requests import urlparse

app = Flask(__name__)
app.secret_key = 'secretkey'
# konfigurasi database
mysql = MySQL()
app.config['MYSQL_DATABASE_USER'] = 'root'
app.config['MYSQL_DATABASE_PASSWORD'] = 'root'
app.config['MYSQL_DATABASE_DB'] = 'dbtst'
app.config['MYSQL_DATABASE_HOST'] = '172.24.0.2'
app.config['MYSQL_DATABASE_PORT'] = 3306
app.config['JSONIFY_PRETTYPRINT_REGULAR'] = True

mysql.init_app(app)

oauth = OAuth()
google = oauth.remote_app('google',
                          base_url='https://www.google.com/accounts/',
                          authorize_url='https://accounts.google.com/o/oauth2/auth',
                          request_token_url=None,
                          request_token_params={'scope': 'https://www.googleapis.com/auth/userinfo.email',
                                                'response_type': 'code'},
                          access_token_url='https://accounts.google.com/o/oauth2/token',
                          access_token_method='POST',
                          access_token_params={'grant_type': 'authorization_code'},
                          consumer_key='482758007120-vm7lob4gqkmr8eeeq21uo5odpnt6736g.apps.googleusercontent.com',
                          consumer_secret='cpNDbgd5rFiw-98asLncKZUd')

@app.route('/')
def landing():
    return render_template('index.html')

@app.route('/home')
def home():
    app.logger.error(time.strftime('%A %B, %d %Y %H:%M:%S') + ' masuk halaman utama')
    access_token = session.get('access_token')
    if access_token is None:
        return redirect(url_for('login'))
    return render_template('home.html')

@app.route('/search')
def search():
    app.logger.error(time.strftime('%A %B, %d %Y %H:%M:%S') + ' masuk halaman pencarian')
    access_token = session.get('access_token')
    if access_token is None:
        return redirect(url_for('login'))
    return render_template('search.html')

@app.route('/register')
def register():
    app.logger.error(time.strftime('%A %B, %d %Y %H:%M:%S') + ' masuk halaman penambahan data')
    access_token = session.get('access_token')
    if access_token is None:
        return redirect(url_for('login'))
    return render_template('register.html')

@app.route('/hapus')
def hapus():
    app.logger.error(time.strftime('%A %B, %d %Y %H:%M:%S') + ' masuk halaman penghapusan data')
    access_token = session.get('access_token')
    if access_token is None:
        return redirect(url_for('login'))
    return render_template('hapus.html')

@app.route('/baru')
def baru():
    app.logger.error(time.strftime('%A %B, %d %Y %H:%M:%S') + ' masuk halaman pembaruan data')
    access_token = session.get('access_token')
    if access_token is None:
        return redirect(url_for('login'))
    return render_template('baru.html')

@app.route('/login')
def login():
    callback=url_for('authorized', _external=True)
    return google.authorize(callback=callback)

@app.route('/authorized')
@google.authorized_handler
def authorized(resp):
    access_token = resp['access_token']
    session['access_token'] = access_token, ''
    return redirect(url_for('home'))

@app.route('/user', methods=['GET'])
def get_user():
    app.logger.error(time.strftime('%A %B, %d %Y %H:%M:%S') + ' mencari user')
    conn = mysql.connect() # connect database
    cursor = conn.cursor() # melakukan perintah
    carinama = request.args.get('carinama')

    query = 'SELECT * FROM mahasiswa WHERE nama=%s'
    data = (carinama)

    cursor.execute(query, data)
    result = cursor.fetchall() # merubah hasil dari my sql menjadi list
    result_baru = []

    # untuk menambahkan key
    for user in result:
       user_baru = {
           'nim': user[0],
           'nama': user[1],
           'jurusan': user[2],
           'angkatan': user[3]
       }
       result_baru.append(user_baru)

    return {'hasil': result_baru}

@app.route('/create', methods=['POST'])
def insert_user():
    app.logger.error(time.strftime('%A %B, %d %Y %H:%M:%S') + ' menambahkan data')
    conn = mysql.connect()
    cursor = conn.cursor()

    nim = request.form['nim']
    nama = request.form['nama']
    jurusan = request.form['jurusan']
    angkatan = request.form['angkatan']

    query = 'INSERT INTO mahasiswa (nim, nama, jurusan, angkatan) VALUES (%s,%s,%s,%s)'
    data = (nim, nama, jurusan, angkatan)

    cursor.execute(query, data)
    conn.commit()
    conn.close()

    return 'data berhasil ditambahkan'

@app.route('/update', methods=['PUT'])
def update_user():
    app.logger.error(time.strftime('%A %B, %d %Y %H:%M:%S') + ' memperbarui data')
    conn = mysql.connect()
    cursor = conn.cursor()

    nim = request.form['nim']
    nama = request.form['nama']
    jurusan = request.form['jurusan']
    angkatan = request.form['angkatan']

    query = 'UPDATE mahasiswa SET nama=%s, jurusan=%s, angkatan=%s WHERE nim=%s'
    data = (nama, jurusan, angkatan, nim)

    cursor.execute(query, data)
    conn.commit()
    conn.close()

    return 'data berhasil diupdate'

@app.route('/delete', methods=['DELETE'])
def delete_user():
    app.logger.error(time.strftime('%A %B, %d %Y %H:%M:%S') + ' menghapus data')
    conn = mysql.connect()
    cursor = conn.cursor()

    query = 'DELETE FROM mahasiswa WHERE nim=%s'
    data = request.form['carinim']

    cursor.execute(query, data)
    conn.commit()
    conn.close()

    return 'data berhasil didelete'

# klo update querynya aja yg diganti. tetep tambah commit execute

if __name__ == "__main__":
    handler = RotatingFileHandler('tst.log', maxBytes=10000, backupCount=1)
    handler.setLevel(logging.INFO)
    app.logger.addHandler(handler)
    app.run(host='0.0.0.0')


def test(path):
  from glob import glob
  from os.path import join
  from shutil import rmtree
  from tempfile import mkdtemp
  from numpy import all, abs
  from quantities import kbar, eV, angstrom
  from pylada.crystal import Structure
  from pylada.vasp import Vasp
  from pylada.vasp.relax import Relax
  from pylada import default_comm

  structure = Structure([[0, 0.5, 0.5],[0.5, 0, 0.5], [0.5, 0.5, 0]], scale=5.43, name='has a name')\
                       .add_atom(0,0,0, "Si")\
                       .add_atom(0.25,0.25,0.25, "Si")

  vasp = Vasp()
  vasp.kpoints    = "Automatic generation\n0\nMonkhorst\n2 2 2\n0 0 0"
  vasp.prec       = "accurate"
  vasp.ediff      = 1e-5
  vasp.encut      = 1
  vasp.ismear     = "fermi"
  vasp.sigma      = 0.01
  vasp.relaxation = "volume"
  vasp.add_specie = "Si", "{0}/pseudos/Si".format(path)
  directory = mkdtemp()
  try:
    functional = Relax(copy=vasp)
    assert abs(functional.ediff - 1e-5) < 1e-8
    assert functional.prec == 'Accurate'
    result = functional(structure, outdir=directory, comm=default_comm,
                        relaxation="volume ionic cellshape")
    assert result.success
    def sortme(a): return int(a.split('/')[-1])
    dirs = sorted(glob(join(join(directory, '*'), '[0-9]')), key=sortme)
  # for previous, current in zip(dirs, dirs[1:]):
  #   assert len(check_output(['diff', join(previous, 'CONTCAR'), join(current, 'POSCAR')])) == 0
  # assert len(check_output(['diff', join(current, 'CONTCAR'), join(directory, 'POSCAR')])) == 0
    assert result.stress.units == kbar and all(abs(result.stress) < 1e0)
    assert result.forces.units == eV/angstrom and all(abs(result.forces) < 1e-1)
    assert result.total_energy.units == eV and all(abs(result.total_energy + 10.668652*eV) < 1e-2)

  finally:
    if directory != '/tmp/test/relax': rmtree(directory)
    pass

if __name__ == "__main__":
  from sys import argv
  test(argv[1])


import spacy
import numpy
import os
from numpy import dot
from numpy.linalg import norm

nlp = spacy.load('en_core_web_lg')

def compare_wordlists_by_spacy_vectors(model, wordlist1, wordlist2):

    vectorlist1 = model(wordlist1)
    vectorlist2 = model(wordlist2) #unicode string
    myData = [ ]

    for wordvec1 in vectorlist1:
        for wordvec2 in vectorlist2:

            myData.append([wordvec1, wordvec2, wordvec1.similarity(wordvec2)])
            #print(wordvec1, wordvec2)
            #print(wordvec1.similarity(wordvec2))
            #print("")
    return  myData

#wordlist1 = (u'illegals refugees immigrants humans')
#wordlist2 = (u'love hate anger disgust')
#compare_wordlists_by_spacy_vectors(nlp, wordlist1, wordlist2)

def compare_within_wordlist_by_spacy_vectors(model, wordlist1):

    vectorlist1 = model(wordlist1)
    vectorlist2 = model(wordlist1) #unicode string
    myData = [ ]

    for wordvec1 in vectorlist1:
        for wordvec2 in vectorlist2:

            myData.append([wordvec1, wordvec2, wordvec1.similarity(wordvec2)])

            #print(wordvec1, wordvec2)
            #print(wordvec1.similarity(wordvec2))
            #print("")
    return myData

#wordlist1 = (u'illegals refugees immigrants humans') #list of strings = input at present
#wordlist2 = (u'love hate anger disgust')
#compare_within_wordlist_by_spacy_vectors(nlp, wordlist1)
#compare_within_wordlist_by_spacy_vectors(nlp, wordlist2)

def similar_strings(model, word):
    '''
    using spaCy .vocab to find top 20 orthographic similar words to an input word
    '''
    myData = [ ]
    # cosine similarity
    cosine = lambda v1, v2: dot(v1, v2) / (norm(v1) * norm(v2))
    # gather all known words, take only the lowercased versions
    allWords = list({w for w in model.vocab if w.has_vector and w.orth_.islower() and w.lower_ != word })
    # sort by similarity to word
    allWords.sort(key=lambda w: cosine(w.vector, word.vector))
    allWords.reverse()
    print("Top 20 most similar words to" , word.orth_,":")
    for word in allWords[:20]:
        myData.append(word.orth_)
    return myData

#hit = nlp.vocab[u'hit']
#similar_strings(nlp, hit)


import sys
import contentful_management

from env import *

client = contentful_management.Client(MANAGEMENT_API_TOKEN)
space = client.spaces().find(SPACE_ID)

environment = space.environments().find('master')
content_types = environment.content_types().all()
content_type = content_types[0]

if '--test' in sys.argv:
    # Test by getting entries and showing local image locations
    print('Testing...')
    print('Your formation index file is set to: ' + FORMATION_INDEX)
    print('Your formation image file directory is set to: ' + FORMATION_IMAGE_DIR)
    entries = content_type.entries().all()

    sys.exit()

if '--delete-all' in sys.argv:
    # Delete all entries
    entries = content_type.entries().all()
    for entry in entries:
        if entry.is_published:
            entry.unpublish()
        environment.entries().delete(entry.id)

    # Delete all assets
    assets = environment.assets().all()
    for asset in assets:
        if asset.is_published:
            asset.unpublish()
        environment.assets().delete(asset.id)

    sys.exit()

with open(FORMATION_INDEX, 'r') as formation_file:
    for line in formation_file:
        formation_code, formation_name = line.strip().split(None, 1)
        formation_size = int(formation_code.split('-')[0])
        formation_filename = formation_code + '.png'

        print(formation_code, formation_name)

        # Create a new upload to get the image into Contentful
        upload = space.uploads().create(FORMATION_IMAGE_DIR + '/' + formation_filename)

        # Associate an asset with the uploaded image
        new_asset = environment.assets().create(
            None,
            {
                'fields': {
                    'title': {
                        'en-US': formation_code + ': ' + formation_name
                    },
                    'file': {
                        'en-US': {
                            'fileName': formation_filename,
                            'contentType': 'image/png',
                            'uploadFrom': upload.to_link().to_json()
                        }
                    }
                }
            }
        )

        # Process the asset
        new_asset.process()

        # Wait for the asset processing to be complete
        while True:
            processed_asset = environment.assets().find(new_asset.id)
            try:
                if 'url' in processed_asset.file:
                    break
            except:
                continue

        # And then publish
        processed_asset.publish()

        # Add an entry that references the asset
        entry_attributes = {
            'content_type_id': content_type.id,
            'fields': {
                'name': {
                    'en-US': formation_name
                },
                'code': {
                    'en-US': formation_code
                },
                'size': {
                    'en-US': formation_size
                },
                'diagram': {
                    'en-US': {
                        'sys': {
                            'type': 'Link',
                            'linkType': 'Asset',
                            'id': processed_asset.id
                        }
                    }
                }
            }
        }

        new_entry = environment.entries().create(
            None,
            entry_attributes
        )

        new_entry.publish()


#date:21-9-17
import pafy
url="https://www.youtube.com/watch?v=mkKXS0FI_L4"
video=pafy.new(url)
audiostreams = video.audiostreams
for a in audiostreams:
    print(a.bitrate, a.extension, a.get_filesize())

########to download audio directly
#audiostreams[1].download()

bestaudio = video.getbestaudio()
bestaudio.bitrate
########To download best audio
#bestaudio.download()

##to see all the streams available
allstreams = video.allstreams
for s in allstreams:
    print(s.mediatype, s.extension, s.quality)


#!/usr/bin/python3
from __future__ import print_function

import os
import sys
import torch
import torch.backends.cudnn as cudnn
import argparse
import cv2
import numpy as np
from collections import OrderedDict

sys.path.append(os.getcwd() + '/../../src')

from config import cfg
from prior_box import PriorBox
from detection import Detect
from nms import nms
from utils import decode
from timer import Timer
from yufacedetectnet import YuFaceDetectNet

parser = argparse.ArgumentParser(description='Face and Mask Detection')
parser.add_argument('-m', '--trained_model', default='weights/yunet_final.pth',
                    type=str, help='Trained state_dict file path to open')
parser.add_argument('--image_file', default='t1.jpg', type=str, help='the image file to be detected')
parser.add_argument('--conf_thresh', default=0.5, type=float, help='conf_thresh')
parser.add_argument('--top_k', default=20, type=int, help='top_k')
parser.add_argument('--nms_thresh', default=0.5, type=float, help='nms_thresh')
parser.add_argument('--keep_top_k', default=20, type=int, help='keep_top_k')
parser.add_argument('--device', default='cuda:0', help='which device the program will run on. cuda:0, cuda:1, ...')
args = parser.parse_args()

def check_keys(model, pretrained_state_dict):
    ckpt_keys = set(pretrained_state_dict.keys())
    model_keys = set(model.state_dict().keys())
    used_pretrained_keys = model_keys & ckpt_keys
    unused_pretrained_keys = ckpt_keys - model_keys
    missing_keys = model_keys - ckpt_keys
    print('Missing keys:{}'.format(len(missing_keys)))
    print('Unused checkpoint keys:{}'.format(len(unused_pretrained_keys)))
    print('Used keys:{}'.format(len(used_pretrained_keys)))
    assert len(used_pretrained_keys) > 0, 'load NONE from pretrained checkpoint'
    return True

def remove_prefix(state_dict, prefix):
    ''' Old style model is stored with all names of parameters sharing common prefix 'module.' '''
    print('remove prefix \'{}\''.format(prefix))
    f = lambda x: x.split(prefix, 1)[-1] if x.startswith(prefix) else x
    return {f(key): value for key, value in state_dict.items()}

def load_model(model, pretrained_path, load_to_cpu):
    print('Loading pretrained model from {}'.format(pretrained_path))
    if load_to_cpu:
        pretrained_dict = torch.load(pretrained_path, map_location=lambda storage, loc: storage)
    else:
        device = torch.cuda.current_device()
        pretrained_dict = torch.load(pretrained_path, map_location=lambda storage, loc: storage.cuda(device))
    if "state_dict" in pretrained_dict.keys():
        pretrained_dict = remove_prefix(pretrained_dict['state_dict'], 'module.')
    else:
        pretrained_dict = remove_prefix(pretrained_dict, 'module.')
    check_keys(model, pretrained_dict)
    model.load_state_dict(pretrained_dict, strict=False)
    return model

labels = ('_background_', 'face', 'mask')
num_classes = 3
detect = Detect(num_classes, 0, args.top_k, args.conf_thresh, args.nms_thresh)

if __name__ == '__main__':
    # img_dim = 320
    device = torch.device(args.device)
    torch.set_grad_enabled(False)

    # net and model
    net = YuFaceDetectNet(phase='test', size=None )    # initialize detector
    net = load_model(net, args.trained_model, True)
    # net.load_state_dict(torch.load(args.trained_model))
    net.eval()
    print('Finished loading model!')

    ## Print model's state_dict
    #print("Model's state_dict:")
    #for param_tensor in net.state_dict():
    #    print(param_tensor, "\t", net.state_dict()[param_tensor].size())

    cudnn.benchmark = True
    net = net.to(device)

    _t = {'forward_pass': Timer(), 'misc': Timer()}
    # testing begin
    img_raw = cv2.imread(args.image_file, cv2.IMREAD_COLOR)
    img = np.float32(img_raw)
    im_height, im_width, _ = img.shape

    #img -= (104, 117, 123)
    img = img.transpose(2, 0, 1)
    img = torch.from_numpy(img).unsqueeze(0)
    img = img.to(device)

    _t['forward_pass'].tic()
    loc, conf = net(img)  # forward pass
    _t['forward_pass'].toc()
    _t['misc'].tic()

    priorbox = PriorBox(cfg, image_size=(im_height, im_width))
    priors = priorbox.forward()
    priors = priors.to(device)

    detections = detect(loc, conf, priors)
    # detections = out.data
    print(detections.size())
    scale = torch.Tensor([im_width, im_height, im_width, im_height])
    # scale = scale.to(device)
    for i in range(detections.size(1)):
        j = 0
        while detections[0,i,j,0] >= 0.6:
            score = detections[0,i,j,0]
            label_name = labels[i]
            display_txt = '%s: %.2f'%(label_name, score)
            pt = (detections[0,i,j,1:]*scale).cpu().numpy()
            j+=1
            pts = (int(pt[0]), int(pt[1]))
            pte = (int(pt[2]), int(pt[3]))
            cx = int(pt[0])
            cy = int(pt[1]) + 12
            cv2.rectangle(img_raw, pts, pte, (0, 255, 0), 2)
            cv2.putText(img_raw, label_name, (cx, cy), cv2.FONT_HERSHEY_DUPLEX, 0.5, (255, 255, 255))
    cv2.imshow('facemask', img_raw)
    cv2.waitKey(0)


import argparse
import os
import statistics
from typing import Callable

import dotenv
from tqdm import tqdm

from headhunter import get_vacancies_from_hh, fetch_areas_ids, predict_rub_salary_hh
from salary_helpers import create_table
from superjob import get_vacancies_from_sj, predict_rub_salary_sj, fetch_town_ids

PROGRAMMING_LANGUAGES = [
    "TypeScript",
    "Swift",
    "Scala",
    "Objective-C",
    "Shell",
    "Go",
    "C",
    "C#",
    "C++",
    "PHP",
    "Ruby",
    "Python",
    "Java",
    "JavaScript",
]

def get_vacancies_statistics(
    vacancies: dict, predict_rub_salary_method: Callable
) -> dict | None:
    salaries = []
    for vacancy in vacancies.get("items"):
        salary = predict_rub_salary_method(vacancy)
        if not salary:
            continue

        salaries.append(salary)

    average_salary = round(statistics.mean(salaries), 0) if salaries else 0

    return {
        "vacancies_found": vacancies.get("found"),
        "vacancies_processed": len(salaries),
        "average_salary": int(average_salary),
    }

def main():
    parser = argparse.ArgumentParser(
        description="Collects statistics on salaries of programming languages."
    )
    parser.add_argument(
        "-l",
        "--location",
        type=str,
        help="Search area. (default: Москва)",
        default="Москва",
    )
    args = parser.parse_args()
    dotenv.load_dotenv()
    salary_statistics = {}
    area_ids = fetch_areas_ids(args.location)
    town_ids = fetch_town_ids(os.getenv("SJ_API_KEY"), args.location)
    for language in tqdm(PROGRAMMING_LANGUAGES):
        vacancy_name = "Программист {}".format(language)
        vacancies = get_vacancies_from_hh(
            text=vacancy_name,
            area_ids=area_ids,
        )
        head_hunter = salary_statistics.setdefault("Head Hunter", {})
        head_hunter[language] = (
            get_vacancies_statistics(vacancies, predict_rub_salary_hh) or {}
        )

        vacancies = get_vacancies_from_sj(
            os.getenv("SJ_API_KEY"), text=vacancy_name, town_ids=town_ids
        )
        super_job = salary_statistics.setdefault("Super Job", {})
        super_job[language] = (
            get_vacancies_statistics(vacancies, predict_rub_salary_sj) or {}
        )

    for platform_title, stat in salary_statistics.items():
        print(create_table(stat, f"{platform_title} {args.location}"))

if __name__ == "__main__":
    main()


from django.db import models
from django.conf import settings

from channels.layers import get_channel_layer
from asgiref.sync import async_to_sync

import uuid

channel_layer = get_channel_layer()

class MessageManager(models.Manager):
    def get_pending_messages(self, user):
        pending_messages_qs = user.pending_messages.order_by('timestamp')
        for message in pending_messages_qs:
            message.remove_user_from_pending(user)
        return pending_messages_qs

    def mark_room_as_read(self, user, room):
        unread_messages_qs = user.unread_messages.filter(room=room)
        for message in unread_messages_qs:
            message.mark_as_read(user)
        return unread_messages_qs

class Message(models.Model):
    author = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        on_delete=models.CASCADE)
    room = models.ForeignKey(
        'Room',
        related_name='messages',
        on_delete=models.CASCADE)
    body = models.TextField(max_length=500, default='')
    timestamp = models.DateTimeField(auto_now_add=True)

    pending_reception = models.ManyToManyField(
        settings.AUTH_USER_MODEL,
        related_name='pending_messages')

    pending_read = models.ManyToManyField(
        settings.AUTH_USER_MODEL,
        related_name='unread_messages')

    front_key = models.UUIDField(
        verbose_name="frontend key",
        default=uuid.uuid4,
        unique=True)

    objects = MessageManager()

    class Meta:
        verbose_name = "Mensaje"
        verbose_name_plural = "Mensajes"

    def __str__(self):
        return self.body

    def signal_to_room(self, message, data={}):
        for participant in self.room.participants.all():
            async_to_sync(channel_layer.group_send)(
                f"group_general_user_{participant.id}", {
                    "type": "chat_message",
                    "message": message,
                    'data': data
                })

    def remove_user_from_pending(self, user):
        if self.pending_reception.filter(id=user.id).exists():
            self.pending_reception.remove(user)
            # If there are no more pending then signal
            if not self.pending_reception.exists():
                self.signal_to_room('update_message', {
                    'message_id': self.id,
                    'kind': 'all_received'
                })

    def mark_as_read(self, user):
        if self.pending_read.filter(id=user.id).exists():
            self.pending_read.remove(user)
            # If there are no more pending then signal
            if not self.pending_read.exists():
                self.signal_to_room('update_message', {
                    'message_id': self.id,
                    'kind': 'all_read'
                })

class Room(models.Model):
    class RoomKind(models.IntegerChoices):
        PRIVATE = 1
        GROUP = 2

    group_name = models.CharField(max_length=255, blank=True, null=True)
    kind = models.IntegerField(choices=RoomKind.choices)
    participants = models.ManyToManyField(
        settings.AUTH_USER_MODEL,
        related_name='rooms')

    created_at = models.DateTimeField(
        verbose_name='Creation Date',
        auto_now_add=True)
    last_activity = models.DateTimeField(
        verbose_name='Last activity date',
        auto_now=True)

    class Meta:
        verbose_name = "Sala"
        verbose_name_plural = "Salas"

    def signal_to_room(self, message, data={}):
        for participant in self.participants.all():
            async_to_sync(channel_layer.group_send)(
                f"group_general_user_{participant.id}", {
                    "type": "chat_message",
                    "message": message,
                    'data': data
                })


import functools
import warnings

import numpy

import cupy
import cupyx.scipy.fft

def _wraps_polyroutine(func):
    def _get_coeffs(x):
        if isinstance(x, cupy.poly1d):
            return x._coeffs
        if cupy.isscalar(x):
            return cupy.atleast_1d(x)
        if isinstance(x, cupy.ndarray):
            x = cupy.atleast_1d(x)
            if x.ndim == 1:
                return x
            raise ValueError('Multidimensional inputs are not supported')
        raise TypeError('Unsupported type')

    def wrapper(*args):
        coeffs = [_get_coeffs(x) for x in args]
        out = func(*coeffs)

        if all(not isinstance(x, cupy.poly1d) for x in args):
            return out
        if isinstance(out, cupy.ndarray):
            return cupy.poly1d(out)
        if isinstance(out, tuple):
            return tuple([cupy.poly1d(x) for x in out])
        assert False  # Never reach

    return functools.update_wrapper(wrapper, func)

def poly(seq_of_zeros):
    """Computes the coefficients of a polynomial with the given roots sequence.

    Args:
        seq_of_zeros (cupy.ndarray): a sequence of polynomial roots.

    Returns:
        cupy.ndarray: polynomial coefficients from highest to lowest degree.

    .. warning::

        This function doesn't support general 2d square arrays currently.
        Only complex Hermitian and real symmetric 2d arrays are allowed.

    .. seealso:: :func:`numpy.poly`

    """
    x = seq_of_zeros
    if x.ndim == 2 and x.shape[0] == x.shape[1] and x.shape[0] != 0:
        if cupy.array_equal(x, x.conj().T):
            x = cupy.linalg.eigvalsh(x)
        else:
            raise NotImplementedError('Only complex Hermitian and real '
                                      'symmetric 2d arrays are supported '
                                      'currently')
    elif x.ndim == 1:
        x = x.astype(cupy.mintypecode(x.dtype.char), copy=False)
    else:
        raise ValueError('Input must be 1d or non-empty square 2d array.')

    if x.size == 0:
        return 1.0

    size = 2 ** (x.size - 1).bit_length()
    a = cupy.zeros((size, 2), x.dtype)
    a[:, 0].fill(1)
    cupy.negative(x, out=a[:x.size, 1])
    while size > 1:
        size = size // 2
        a = cupy._math.misc._fft_convolve(a[:size], a[size:], 'full')
    return a[0, :x.size + 1]

@_wraps_polyroutine
def polyadd(a1, a2):
    """Computes the sum of two polynomials.

    Args:
        a1 (scalar, cupy.ndarray or cupy.poly1d): first input polynomial.
        a2 (scalar, cupy.ndarray or cupy.poly1d): second input polynomial.

    Returns:
        cupy.ndarray or cupy.poly1d: The sum of the inputs.

    .. seealso:: :func:`numpy.polyadd`

    """
    if a1.size < a2.size:
        a1, a2 = a2, a1
    out = cupy.pad(a2, (a1.size - a2.size, 0))
    out = out.astype(cupy.result_type(a1, a2), copy=False)
    out += a1
    return out

@_wraps_polyroutine
def polysub(a1, a2):
    """Computes the difference of two polynomials.

    Args:
        a1 (scalar, cupy.ndarray or cupy.poly1d): first input polynomial.
        a2 (scalar, cupy.ndarray or cupy.poly1d): second input polynomial.

    Returns:
        cupy.ndarray or cupy.poly1d: The difference of the inputs.

    .. seealso:: :func:`numpy.polysub`

    """
    if a1.shape[0] <= a2.shape[0]:
        out = cupy.pad(a1, (a2.shape[0] - a1.shape[0], 0))
        out = out.astype(cupy.result_type(a1, a2), copy=False)
        out -= a2
    else:
        out = cupy.pad(a2, (a1.shape[0] - a2.shape[0], 0))
        out = out.astype(cupy.result_type(a1, a2), copy=False)
        out -= 2 * out - a1
    return out

@_wraps_polyroutine
def polymul(a1, a2):
    """Computes the product of two polynomials.

    Args:
        a1 (scalar, cupy.ndarray or cupy.poly1d): first input polynomial.
        a2 (scalar, cupy.ndarray or cupy.poly1d): second input polynomial.

    Returns:
        cupy.ndarray or cupy.poly1d: The product of the inputs.

    .. seealso:: :func:`numpy.polymul`

    """
    a1 = cupy.trim_zeros(a1, trim='f')
    a2 = cupy.trim_zeros(a2, trim='f')
    if a1.size == 0:
        a1 = cupy.array([0.], a1.dtype)
    if a2.size == 0:
        a2 = cupy.array([0.], a2.dtype)
    return cupy.convolve(a1, a2)

def _polypow_direct(x, n):
    if n == 0:
        return 1
    if n == 1:
        return x
    if n % 2 == 0:
        return _polypow(cupy.convolve(x, x), n // 2)
    return cupy.convolve(x, _polypow(cupy.convolve(x, x), (n - 1) // 2))

def _polypow(x, n):
    if n == 0:
        return 1
    if n == 1:
        return x

    method = cupy._math.misc._choose_conv_method(x, x, 'full')

    if method == 'direct':
        return _polypow_direct(x, n)
    elif method == 'fft':
        if x.dtype.kind == 'c':
            fft, ifft = cupy.fft.fft, cupy.fft.ifft
        else:
            fft, ifft = cupy.fft.rfft, cupy.fft.irfft
        out_size = (x.size - 1) * n + 1
        size = cupyx.scipy.fft.next_fast_len(out_size)
        fx = fft(x, size)
        fy = cupy.power(fx, n, fx)
        y = ifft(fy, size)
        return y[:out_size]
    else:
        assert False

def _polyfit_typecast(x):
    if x.dtype.kind == 'c':
        return x.astype(numpy.complex128, copy=False)
    return x.astype(numpy.float64, copy=False)

def polyfit(x, y, deg, rcond=None, full=False, w=None, cov=False):
    """Returns the least squares fit of polynomial of degree deg
    to the data y sampled at x.

    Args:
        x (cupy.ndarray): x-coordinates of the sample points of shape (M,).
        y (cupy.ndarray): y-coordinates of the sample points of shape
            (M,) or (M, K).
        deg (int): degree of the fitting polynomial.
        rcond (float, optional): relative condition number of the fit.
            The default value is ``len(x) * eps``.
        full (bool, optional): indicator of the return value nature.
            When False (default), only the coefficients are returned.
            When True, diagnostic information is also returned.
        w (cupy.ndarray, optional): weights applied to the y-coordinates
            of the sample points of shape (M,).
        cov (bool or str, optional): if given, returns the coefficients
            along with the covariance matrix.

    Returns:
        cupy.ndarray or tuple:
        p (cupy.ndarray of shape (deg + 1,) or (deg + 1, K)):
            Polynomial coefficients from highest to lowest degree
        residuals, rank, singular_values, rcond \
        (cupy.ndarray, int, cupy.ndarray, float):
            Present only if ``full=True``.
            Sum of squared residuals of the least-squares fit,
            rank of the scaled Vandermonde coefficient matrix,
            its singular values, and the specified value of ``rcond``.
        V (cupy.ndarray of shape (M, M) or (M, M, K)):
            Present only if ``full=False`` and ``cov=True``.
            The covariance matrix of the polynomial coefficient estimates.

    .. warning::

        numpy.RankWarning: The rank of the coefficient matrix in the
        least-squares fit is deficient. It is raised if ``full=False``.

    .. seealso:: :func:`numpy.polyfit`

    """
    if x.dtype.char == 'e' and y.dtype.kind == 'b':
        raise NotImplementedError('float16 x and bool y are not'
                                  ' currently supported')
    if y.dtype == numpy.float16:
        raise TypeError('float16 y are not supported')

    x = _polyfit_typecast(x)
    y = _polyfit_typecast(y)
    deg = int(deg)

    if deg < 0:
        raise ValueError('expected deg >= 0')
    if x.ndim != 1:
        raise TypeError('expected 1D vector for x')
    if x.size == 0:
        raise TypeError('expected non-empty vector for x')
    if y.ndim < 1 or y.ndim > 2:
        raise TypeError('expected 1D or 2D array for y')
    if x.size != y.shape[0]:
        raise TypeError('expected x and y to have same length')

    lhs = cupy.polynomial.polynomial.polyvander(x, deg)[:, ::-1]
    rhs = y

    if w is not None:
        w = _polyfit_typecast(w)
        if w.ndim != 1:
            raise TypeError('expected a 1-d array for weights')
        if w.size != x.size:
            raise TypeError('expected w and y to have the same length')

        lhs *= w[:, None]
        if rhs.ndim == 2:
            w = w[:, None]
        rhs *= w

    if rcond is None:
        rcond = x.size * cupy.finfo(x.dtype).eps

    scale = cupy.sqrt((cupy.square(lhs)).sum(axis=0))
    lhs /= scale
    c, resids, rank, s = cupy.linalg.lstsq(lhs, rhs, rcond)
    if y.ndim > 1:
        scale = scale.reshape(-1, 1)
    c /= scale

    order = deg + 1
    if rank != order and not full:
        msg = 'Polyfit may be poorly conditioned'
        warnings.warn(msg, numpy.RankWarning, stacklevel=4)

    if full:
        if resids.dtype.kind == 'c':
            resids = cupy.absolute(resids)
        return c, resids, rank, s, rcond
    if cov:
        base = cupy.linalg.inv(cupy.dot(lhs.T, lhs))
        base /= cupy.outer(scale, scale)

        if cov == 'unscaled':
            factor = 1
        elif x.size > order:
            factor = resids / (x.size - order)
        else:
            raise ValueError('the number of data points must exceed order'
                             ' to scale the covariance matrix')

        if y.ndim != 1:
            base = base[..., None]
        return c, base * factor

    return c

def polyval(p, x):
    """Evaluates a polynomial at specific values.

    Args:
        p (cupy.ndarray or cupy.poly1d): input polynomial.
        x (scalar, cupy.ndarray): values at which the polynomial
        is evaluated.

    Returns:
        cupy.ndarray or cupy.poly1d: polynomial evaluated at x.

    .. warning::

        This function doesn't currently support poly1d values to evaluate.

    .. seealso:: :func:`numpy.polyval`

    """
    if isinstance(p, cupy.poly1d):
        p = p.coeffs
    if not isinstance(p, cupy.ndarray) or p.ndim == 0:
        raise TypeError('p must be 1d ndarray or poly1d object')
    if p.ndim > 1:
        raise ValueError('p must be 1d array')
    if isinstance(x, cupy.poly1d):
        # TODO(asi1024): Needs performance improvement.
        dtype = numpy.result_type(x.coeffs, 1)
        res = cupy.poly1d(cupy.array([0], dtype=dtype))
        prod = cupy.poly1d(cupy.array([1], dtype=dtype))
        for c in p[::-1]:
            res = res + prod * c
            prod = prod * x
        return res
    dtype = numpy.result_type(p.dtype.type(0), x)
    p = p.astype(dtype, copy=False)
    if p.size == 0:
        return cupy.zeros(x.shape, dtype)
    if dtype == numpy.bool_:
        return p.any() * x + p[-1]
    if not cupy.isscalar(x):
        x = cupy.asarray(x, dtype=dtype)[..., None]
    x = x ** cupy.arange(p.size, dtype=dtype)
    return (p[::-1] * x).sum(axis=-1, dtype=dtype)

def roots(p):
    """Computes the roots of a polynomial with given coefficients.

    Args:
        p (cupy.ndarray or cupy.poly1d): polynomial coefficients.

    Returns:
        cupy.ndarray: polynomial roots.

    .. warning::

        This function doesn't support currently polynomial coefficients
        whose companion matrices are general 2d square arrays. Only those
        with complex Hermitian or real symmetric 2d arrays are allowed.

        The current `cupy.roots` doesn't guarantee the order of results.

    .. seealso:: :func:`numpy.roots`

    """
    if isinstance(p, cupy.poly1d):
        p = p.coeffs
    if p.dtype.kind == 'b':
        raise NotImplementedError('boolean inputs are not supported')
    if p.ndim == 0:
        raise TypeError('0-dimensional input is not allowed')
    if p.size < 2:
        return cupy.array([])
    [p] = cupy.polynomial.polyutils.as_series([p[::-1]])
    if p.size < 2:
        return cupy.array([])
    if p.size == 2:
        out = (-p[0] / p[1])[None]
        if p[0] == 0:
            out = out.real.astype(numpy.float64)
        return out
    cmatrix = cupy.polynomial.polynomial.polycompanion(p)
    # TODO(Dahlia-Chehata): Support after cupy.linalg.eigvals is supported
    if cupy.array_equal(cmatrix, cmatrix.conj().T):
        out = cupy.linalg.eigvalsh(cmatrix)
    else:
        raise NotImplementedError('Only complex Hermitian and real '
                                  'symmetric 2d arrays are supported '
                                  'currently')
    return out.astype(p.dtype)


import pytest
from hamcrest import assert_that, equal_to

from gairl.memory.prioritized_replay_buffer import _SumTree

def test_init_valid():
    # When
    tree = _SumTree(8)

    # Then
    assert_that(tree.total_priority, equal_to(0))
    assert_that(tree.priorities_range, equal_to((1, 1)))
    assert_that(tree._data, equal_to([0]*8))
    assert_that(tree._tree, equal_to([0]*15))

def test_init_capacity_not_power_2():
    # When / Then
    with pytest.raises(AssertionError):
        _SumTree(10)

def test_add_not_full():
    # Given
    tree = _SumTree(16)

    # When
    tree.add((1, 'a', 1.), 1)
    tree.add(('b', 2, 2.), 0.1)
    tree.add(195, 3)
    tree.add((3, 3., 'c'), 1)
    tree.add('d', 5)
    tree.add(19287412.214121, 0.1)
    tree.add(0, 9)

    # Then
    assert_that(tree.priorities_range, equal_to((0.1, 9)))
    assert_that(tree._max_priorities_num, equal_to(1))
    assert_that(tree._min_priorities_num, equal_to(2))
    assert_that(tree._data, equal_to([
        (1, 'a', 1.), ('b', 2, 2.), 195,
        (3, 3., 'c'), 'd', 19287412.214121, 0,
        0, 0, 0, 0, 0, 0, 0, 0, 0
    ]))
    assert_that(tree._tree, equal_to([
        19.2,
        19.2, 0,
        5.1, 14.1, 0, 0,
        1.1, 4, 5.1, 9, 0, 0, 0, 0,
        1, 0.1, 3, 1, 5, 0.1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0
    ]))

def test_add_overflow():
    # Given
    tree = _SumTree(4)

    # When
    tree.add((1, 'a', 1.), 1)
    tree.add(('b', 2, 2.), 0.1)
    tree.add(195, 3)
    tree.add((3, 3., 'c'), 1)
    tree.add('d', 5)
    tree.add(19287412.214121, 0.1)
    tree.add(0, 9)

    # Then
    assert_that(tree.priorities_range, equal_to((0.1, 9)))
    assert_that(tree._max_priorities_num, equal_to(1))
    assert_that(tree._min_priorities_num, equal_to(1))
    assert_that(tree._data, equal_to(['d', 19287412.214121, 0, (3, 3., 'c')]))
    assert_that(tree._tree, equal_to([15.1, 5.1, 10, 5, 0.1, 9, 1]))

def test_get_not_full():
    # Given
    tree = _SumTree(16)
    tree._data = [
        (1, 'a', 1.), ('b', 2, 2.), 195,
        (3, 3., 'c'), 'd', 19287412.214121, 0,
        0, 0, 0, 0, 0, 0, 0, 0, 0
    ]
    tree._tree = [
        22,
        22, 0,
        7, 15, 0, 0,
        4, 3, 6, 9, 0, 0, 0, 0,
        1, 3, 1, 2, 1, 5, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0
    ]

    # When
    data1 = tree.get_data(9.3)
    data2 = tree.get_data(10.7)
    data3 = tree.get_data(5.1)
    data4 = tree.get_data(0)
    data5 = tree.get_data(22)
    data6 = tree.get_data(13.001)
    data7 = tree.get_data(1.9)

    # Then
    assert_that(data1, equal_to((19287412.214121, 5, 5)))
    assert_that(data2, equal_to((19287412.214121, 5, 5)))
    assert_that(data3, equal_to(((3, 3., 'c'), 3, 2)))
    assert_that(data4, equal_to(((1, 'a', 1.), 0, 1)))
    assert_that(data5, equal_to((0, 6, 9)))
    assert_that(data6, equal_to((0, 6, 9)))
    assert_that(data7, equal_to((('b', 2, 2.), 1, 3)))

def test_get_overflow():
    # Given
    tree = _SumTree(4)
    tree._data = ['d', 19287412.214121, 0, (3, 3., 'c')]
    tree._tree = [
        15,
        5, 10,
        4, 1, 7, 3
    ]

    # When
    data1 = tree.get_data(0.31)
    data2 = tree.get_data(4.7)
    data3 = tree.get_data(11.9999)
    data4 = tree.get_data(12.1)
    data5 = tree.get_data(15)

    # Then
    assert_that(data1, equal_to(('d', 0, 4)))
    assert_that(data2, equal_to((19287412.214121, 1, 1)))
    assert_that(data3, equal_to((0, 2, 7)))
    assert_that(data4, equal_to(((3, 3., 'c'), 3, 3)))
    assert_that(data5, equal_to(((3, 3., 'c'), 3, 3)))

def test_get_higher_than_total():
    # Given
    tree = _SumTree(4)
    tree._data = ['d', 19287412.214121, 0, (3, 3., 'c')]
    tree._tree = [
        15,
        5, 10,
        4, 1, 7, 3
    ]

    # When
    with pytest.raises(AssertionError):
        tree.get_data(15.001)

def test_update_priority_no_maxmin_change():
    # Given
    tree = _SumTree(8)
    tree._data = ['d', 19287412.214121, 0, (3, 3., 'c')]
    tree._tree = [
        16,
        6, 10,
        4, 2, 7, 3,
        1, 3, 1, 1, 4, 3, 1, 2
    ]
    tree._min_priority = 1
    tree._min_priorities_num = 4
    tree._max_priority = 4
    tree._max_priorities_num = 1

    # When
    tree.update_priority(0, 2)
    tree.update_priority(2, 2)
    tree.update_priority(5, 2)
    tree.update_priority(6, 2)

    # Then
    assert_that(tree.priorities_range, equal_to((1, 4)))
    assert_that(tree._max_priorities_num, equal_to(1))
    assert_that(tree._min_priorities_num, equal_to(1))
    assert_that(tree._data, equal_to(['d', 19287412.214121, 0, (3, 3., 'c')]))
    assert_that(tree._tree, equal_to([
        18,
        8, 10,
        5, 3, 6, 4,
        2, 3, 2, 1, 4, 2, 2, 2
    ]))

def test_update_priority_maxmin_run_out():
    # Given
    tree = _SumTree(8)
    tree._data = ['d', 19287412.214121, 0, (3, 3., 'c')]
    tree._tree = [
        18,
        8, 10,
        5, 3, 7, 3,
        2, 3, 1, 2, 4, 3, 1, 2
    ]
    tree._min_priority = 1
    tree._min_priorities_num = 2
    tree._max_priority = 4
    tree._max_priorities_num = 1

    # When
    tree.update_priority(4, 3)
    tree.update_priority(2, 2)
    tree.update_priority(6, 3)
    tree.update_priority(1, 3)
    tree.update_priority(3, 2)
    tree.update_priority(5, 2)

    # Then
    assert_that(tree.priorities_range, equal_to((2, 3)))
    assert_that(tree._max_priorities_num, equal_to(3))
    assert_that(tree._min_priorities_num, equal_to(5))
    assert_that(tree._data, equal_to(['d', 19287412.214121, 0, (3, 3., 'c')]))
    assert_that(tree._tree, equal_to([
        19,
        9, 10,
        5, 4, 5, 5,
        2, 3, 2, 2, 3, 2, 3, 2
    ]))

def test_update_priority_maxmin_overwrite():
    tree = _SumTree(8)
    tree._data = ['d', 19287412.214121, 0, (3, 3., 'c')]
    tree._tree = [
        18,
        8, 10,
        5, 3, 7, 3,
        2, 3, 1, 2, 4, 3, 1, 2
    ]
    tree._min_priority = 1
    tree._min_priorities_num = 2
    tree._max_priority = 4
    tree._max_priorities_num = 1

    # When
    tree.update_priority(1, 5)
    tree.update_priority(4, 0.5)
    tree.update_priority(3, 1)
    tree.update_priority(7, 5)

    # Then
    assert_that(tree.priorities_range, equal_to((0.5, 5)))
    assert_that(tree._max_priorities_num, equal_to(2))
    assert_that(tree._min_priorities_num, equal_to(1))
    assert_that(tree._data, equal_to(['d', 19287412.214121, 0, (3, 3., 'c')]))
    assert_that(tree._tree, equal_to([
        18.5,
        9, 9.5,
        7, 2, 3.5, 6,
        2, 5, 1, 1, 0.5, 3, 1, 5
    ]))


import os
from pathlib import Path

import structlog

# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve(strict=True).parent.parent.parent

INSTALLED_APPS = [
    "django.contrib.admin",
    "django.contrib.auth",
    "django.contrib.contenttypes",
    "django.contrib.sessions",
    "django.contrib.messages",
    "django.contrib.staticfiles",
    "rest_framework",
    "rest_framework_api_key",
    "drf_spectacular",
    "django_assets",
    "django_celery_beat",
    "django_celery_results",
    "django_extensions",
    "django_filters",
    "django_admin_filters",
    "crispy_forms",
    "jsonify",
    "import_export",
    "gerrit",
    "tracker",
    "hotfix",
    "panel",
    "release_dashboard",
    "build",
    "buildinfo",
    "release_changed",
    "repoapi",
]

MIDDLEWARE = (
    "django.contrib.sessions.middleware.SessionMiddleware",
    "django.middleware.common.CommonMiddleware",
    "django.middleware.csrf.CsrfViewMiddleware",
    "django.contrib.auth.middleware.AuthenticationMiddleware",
    "django.contrib.messages.middleware.MessageMiddleware",
    "django.middleware.clickjacking.XFrameOptionsMiddleware",
    "django.middleware.security.SecurityMiddleware",
    "django_structlog.middlewares.RequestMiddleware",
)

DJANGO_STRUCTLOG_CELERY_ENABLED = True

ROOT_URLCONF = "repoapi.urls"
LOGIN_URL = "rest_framework:login"
LOGOUT_URL = "rest_framework:logout"
TEMPLATES = [
    {
        "BACKEND": "django.template.backends.django.DjangoTemplates",
        "DIRS": ["repoapi/templates"],
        "APP_DIRS": True,
        "OPTIONS": {
            "context_processors": [
                "django.template.context_processors.debug",
                "django.template.context_processors.request",
                "django.contrib.auth.context_processors.auth",
                "django.contrib.messages.context_processors.messages",
            ],
        },
    },
]

WSGI_APPLICATION = "repoapi.wsgi.application"

# Internationalization
# https://docs.djangoproject.com/en/1.8/topics/i18n/

LANGUAGE_CODE = "en-us"

TIME_ZONE = "UTC"

USE_I18N = True

USE_L10N = True

USE_TZ = True

DEFAULT_AUTO_FIELD = "django.db.models.AutoField"

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.8/howto/static-files/

STATIC_URL = "/static/"

STATICFILES_FINDERS = (
    "django.contrib.staticfiles.finders.FileSystemFinder",
    "django.contrib.staticfiles.finders.AppDirectoriesFinder",
    "django_assets.finders.AssetsFinder",
)

STATIC_ROOT = BASE_DIR / "static_media/"

REST_FRAMEWORK = {
    "PAGE_SIZE": 10,
    "DEFAULT_PAGINATION_CLASS": "rest_framework"
    ".pagination.LimitOffsetPagination",
    "DEFAULT_FILTER_BACKENDS": (
        "django_filters.rest_framework.DjangoFilterBackend",
    ),
    "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema",
}

SPECTACULAR_SETTINGS = {
    "TITLE": "RepoApi",
    "DESCRIPTION": "repoapi, one ring to rule them all",
    "VERSION": "1.0.0",
    "CONTACT": {
        "email": "development@sipwise.com",
        "url": "https://www.sipwise.com/",
    },
    "LICENSE": {
        "name": "GPLv3",
        "url": "https://www.gnu.org/licenses/gpl-3.0.en.html",
    },
}

LOGGING = {
    "version": 1,
    "disable_existing_loggers": False,
    "formatters": {
        "plain_console": {
            "()": structlog.stdlib.ProcessorFormatter,
            "processor": structlog.dev.ConsoleRenderer(),
        },
    },
    "handlers": {
        "console": {
            "class": "logging.StreamHandler",
            "formatter": "plain_console",
        }
    },
    "loggers": {
        "django_structlog": {
            "handlers": ["console"],
            "level": "INFO",
            "propagate": False,
        },
        "repoapi": {
            "handlers": ["console"],
            "level": os.getenv("DJANGO_LOG_LEVEL", "INFO"),
        },
    },
}

JENKINS_TOKEN = "sipwise_jenkins_ci"
CRISPY_TEMPLATE_PACK = "bootstrap3"
CELERY_TASK_SERIALIZER = "json"
CELERY_RESULT_SERIALIZER = "json"
CELERY_ACCEPT_CONTENT = ["application/json"]
CELERY_RESULT_BACKEND = "django-db"
CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler"

API_KEY_CUSTOM_HEADER = "HTTP_API_KEY"


# Imports
import PySimpleGUI as sg
import openai
import os
from openai.error import APIConnectionError, AuthenticationError
import threading

from datetime import datetime

from peewee import SqliteDatabase, Model, CharField, TextField

# Envs
from dotenv import load_dotenv, find_dotenv
_ = load_dotenv(find_dotenv())

openai.api_key  = os.getenv('openai.api_key')

# Definitions
APP_TITLE="GUI para Chat GPT"

FILE="Archivo"
RESET_CONTEXT="Restablecer contexto"
SAVE_CHAT="Exportar chat"
QUIT="Salir"

OPTIONS="Opciones"
LOAD_API_KEY="Cargar API Key"

HELP="Ayuda"
ABOUT="Acerca de"

DEFAULT_THEME="DarkGrey1"

CHAT_RESULT_KEY="-chat_result-"
PROMPT_KEY="-prompt_input-"
SUBMIT_KEY="-send_prompt-"
CHAT_LISTBOX = "-select-a-chat"
DELETE_CHAT_BUTTON = "-delete-chat-button"
REGENERATE_CHAT_BUTTON = "-regenerate-chat-button"

THEMES=["Temas", ["Default", "Black", "Dark15", "DarkGrey2", "DarkGrey3", "DarkBrown1"]]

ABOUT_TEXT = "ACERCA DE "

# Define  database connection
db = SqliteDatabase('my_chats.db')

# Define table
class Chat(Model):
    title = CharField()
    query = TextField()
    response = TextField()

    class Meta:
        database = db

# Migrate
db.create_tables([Chat])

# Program
class Application:
    def __init__(self):
        # Tema por defecto
        sg.theme(DEFAULT_THEME)

        # Lista de chats
        self.chats = Chat.select()

        # Contexto predeterminado
        self.default_context = {"role": "system",
           "content": "Eres un asistente muy útil."}

        # Mensajes
        self.messages = [self.default_context]

        # Crea los elementos de la barra de menus
        self.menu_def = [
            [FILE, [RESET_CONTEXT, SAVE_CHAT, "---", QUIT]],
            [OPTIONS, [THEMES, LOAD_API_KEY]],
            [HELP, [ABOUT]]
        ]
        self.menu_bar = sg.Menu(self.menu_def)

        # Left frame
        self.chat_list = sg.Listbox(values=list(map(lambda c : c.title, self.chats)), size=(25, 10), expand_y=True, enable_events=True, key=CHAT_LISTBOX)
        self.new_chat_button = sg.Button("Regenerar consulta", key=REGENERATE_CHAT_BUTTON)
        self.delete_chat_button = sg.Button("Eliminar", key=DELETE_CHAT_BUTTON)
        self.left_frame_layout = [[self.chat_list],[self.new_chat_button, self.delete_chat_button]]
        self.left_frame = sg.Frame(title="Historial de conversaciones", layout=self.left_frame_layout, expand_y=True)

        # Crea los elementos del panel de la derecha
        self.chat_result = sg.Multiline(size=(100, 25), key=CHAT_RESULT_KEY)
        self.prompt_label = sg.Text("Sobre qué quieres hablar?:")
        self.prompt = sg.Multiline(key=PROMPT_KEY, expand_x=True, size=(100, 5))
        self.submit_button = sg.Button("Enviar", key=SUBMIT_KEY, enable_events=True, bind_return_key=True, expand_x=True)
        self.right_frame_layout = [
            [self.chat_result],
            [self.prompt_label],
            [self.prompt],
            [self.submit_button]
        ]
        self.right_frame = sg.Frame(title="Conversación", layout=self.right_frame_layout)

        # Crea la ventana
        self.layout = [
            [self.menu_bar],
            [self.left_frame, sg.VerticalSeparator(), self.right_frame]
        ]
        self.window = sg.Window(APP_TITLE, self.layout)

    # Inicia un bucle para manejar los eventos de la ventana
    def start(self):
        first_load = True
        while True:
            # Leer eventos
            event, values = self.window.read()

            if first_load:
                self.refresh_chat_list()
                first_load = False

            # Cierra la ventana
            if event == sg.WIN_CLOSED or event == QUIT:
                break

            # Click en Enviar
            if event in (SUBMIT_KEY, 'Return:'+PROMPT_KEY):

                # Si necesita clave de la api
                if self.needs_api_key():
                    # Informar que tiene que cargar una clave
                    sg.popup("No se cargó ninguna API Key", "No se cargó ninguna API Key. Para conseguir una visita https://platform.openai.com\nLuego puedes cargarla a través de Opciones>Cargar API Key")
                else:
                    # Obtener la consulta del usuario
                    query = values[PROMPT_KEY]
                    # Se pasa al método encargado de procesar la consulta
                    self.send_query(query)
                    # Limpiar el cuadro de consultas
                    self.window[PROMPT_KEY].update(value="")
            # Cargar API KEY
            elif event == LOAD_API_KEY:
                self.load_api_key()
            # Guardar conversacion en un archivo
            elif event == SAVE_CHAT:
                # Se solicita al usuario que escoga donde guardar
                filename = sg.tk.filedialog.asksaveasfilename(
                    defaultextension='txt',
                    filetypes=(("Archivos de texto", "*.txt"), ("Todos los archivos", "*.*")),
                    parent=self.window.TKroot,
                    title="Guardar como",
                    initialfile=self.chat_list.get()[0]
                )
                # Si se escogió un archivo
                if filename != None and len(filename) > 0:
                    # Se exporta la conversacion
                    self.save_chat_to(filename)
            # Acerca de
            elif event == ABOUT:
                # Muestra cuadro acerca de
                sg.popup(ABOUT_TEXT)
            # Nuevo chat
            elif event == RESET_CONTEXT:
                self.reset_context()
            elif event == CHAT_LISTBOX:
                # Si la lista tiene al menos un elemento
                if self.chat_list.get():
                    # Obtener el elemento seleccionado
                    selected_title = self.chat_list.get()[0]
                    # Cargar chat
                    self.load_chat(selected_title)
                    # Obtener el indice
                    index = self.chat_list.get_list_values().index(selected_title)
                    # Establecer como seleccionado
                    self.chat_list.update(set_to_index=index)

            elif event == DELETE_CHAT_BUTTON:
                delete = sg.popup_yes_no("Desea eliminar la conversación seleccionada?")
                if delete == "Yes":
                    self.delete_chat(self.chat_list.get()[0])

            elif event == REGENERATE_CHAT_BUTTON:
                self.regenerate_query(self.chat_list.get()[0])

        # Destruir/cerrar la ventana cuando finaliza el bucle
        self.window.close()

    # Procesar una consulta
    def send_query(self, query):
        # Crea un nuevo recurso
        new_chat = Chat(title=query[:45]+str(datetime.now()), query=query, response="Esperando respuesta")
        # Guarda en la db
        new_chat.save()
        # Delegacion de la carga
        self.load_chat(new_chat.title)
        # Crear un hilo para escuchar la respuesta del servidor sin bloquear la app
        threading.Thread(target=self.push_response, args=[new_chat.title, query]).start()

    def set_query_response(self, title, response):
        # Busca por query
        selected_chat = Chat.get(Chat.title == title)
        # Actualiza la respuesta
        selected_chat.response = response
        # Guarda
        selected_chat.save()
        # Carga/muestra el resultado
        self.load_chat(title)

    def delete_chat(self, title):
        selected_chat = Chat.get(Chat.title == title)
        selected_chat.delete_instance()
        self.refresh_chat_list()

    def load_chat(self, title):
        # Buscar consulta
        chat_from_db = Chat.get(Chat.title == title)
        # Armar el texto
        chat_text = f"Usuario: {chat_from_db.query}\n"
        chat_text += f"ChatBot: {chat_from_db.response}\n"
        # Mostrar el texto en el chat view/result
        self.window[CHAT_RESULT_KEY].update(value=chat_text)
        self.refresh_chat_list()

    def refresh_chat_list(self):
        # Recargar la lista de chats
        self.chats = Chat.select()
        # Actualizar el listbox con los titulos
        self.chat_list.update(values=[chat.title for chat in self.chats], set_to_index=len(self.chats)-1)

    def regenerate_query(self, title):
        # Buscamos el chat en la db
        selected_chat = Chat.get(Chat.title == title)
        # Y volvemos a enviar la consulta
        self.send_query(selected_chat.query)

    def load_api_key(self):
        # Solicitar al usuario que ingrese la clave mediante ventana emergente
        new_api_key = sg.popup_get_text(title="Cargar API Key", message="Pega aquí tu API Key:", default_text=openai.api_key)
        # Si la nueva clave es ingresada, se guarda, sino se mantiene la anterior
        openai.api_key = new_api_key if new_api_key != None else openai.api_key

        with open(".env", "w") as file:
            file.write(f"openai.api_key={openai.api_key}")

    # Nuevo chat
    def create_new_chat(self, title, content):
        new_chat = {
            "title": title,
            "messages": content
        }
        self.chats.append(new_chat)
        self.chat_list.update(values=[chat.title for chat in self.chats])

    # Reiniciar chat
    def reset_context(self):
        self.messages = [self.default_context]
        self.window[CHAT_RESULT_KEY].update(value="")
        self.window[PROMPT_KEY].update(value="")

    # Añadir texto al chat
    def push_to_chat(self, name, text):

        if len(self.chats) == 0:
            self.create_new_chat(text[:20], text)

        chat = self.window[CHAT_RESULT_KEY].get()
        chat += "\n" if chat != "" else "" # Ahre
        chat += name
        chat += ": "
        chat += text
        chat += "\n"
        self.window[CHAT_RESULT_KEY].update(value=chat)

    def push_response(self, title, query):
        try:
            # Se agrega la consulta al contexto
            self.messages.append({"role": "user", "content": query})
            # Se envía la consulta
            response = openai.ChatCompletion.create(
                model="gpt-3.5-turbo", messages=self.messages)
            # Se saca lo importante de la respuesta
            response_content = response.choices[0].message.content
            # Lo agregamos al contexto
            self.messages.append({"role": "assistant", "content": response_content})
            # Lo asociamos con la consulta
            self.set_query_response(title, response_content)
        except APIConnectionError as ace:
            self.push_to_chat("Sistema", "Ocurrió un error al conectarse con el servidor. Asegurate de que tienes accesso a internet")
        except AuthenticationError as authEx:
            self.push_to_chat("Sistema", "Error de autenticación. Asegúrese de haber proporcionado una API KEY válida")

    def needs_api_key(self):
        return openai.api_key == ""

    def save_chat_to(self, filename):
        with open(filename, "w") as file:
            file.write(self.window[CHAT_RESULT_KEY].get())

app = Application()
app.start()


import tensorflow as tf
import utils  # local file import from baselines.toxic_comments

class UtilsTest(tf.test.TestCase):

  def test_make_cv_train_and_eval_splits(self):
    num_folds = 10
    train_fold_ids = ['2', '5']

    (train_split, eval_split, train_folds, eval_folds,
     eval_fold_ids) = utils.make_cv_train_and_eval_splits(
         num_folds, train_fold_ids, return_individual_folds=True)

    expected_train_folds = ['train[20%:30%]', 'train[50%:60%]']
    expected_eval_folds = [
        'train[0%:10%]', 'train[10%:20%]', 'train[30%:40%]', 'train[40%:50%]',
        'train[60%:70%]', 'train[70%:80%]', 'train[80%:90%]', 'train[90%:100%]'
    ]
    expected_eval_fold_ids = [0, 1, 3, 4, 6, 7, 8, 9]

    self.assertEqual(train_split, 'train[20%:30%]+train[50%:60%]')
    self.assertEqual(eval_split, '+'.join(expected_eval_folds))
    self.assertListEqual(train_folds, expected_train_folds)
    self.assertListEqual(eval_folds, expected_eval_folds)
    self.assertListEqual(eval_fold_ids, expected_eval_fold_ids)

if __name__ == '__main__':
  tf.test.main()


import json
import logging
from .codebase import CodebaseAgent
from .gpt_agent import GPTAgent, Role

class ProgrammerAgent:
    def __init__(self, codebase_repo_path, gpt_api_key):
        self.codebase_agent = CodebaseAgent(codebase_repo_path)
        self.gpt_agent = GPTAgent(api_key=gpt_api_key, role=Role.PROGRAMMER,enable_memory=True)

    def get_code(self, task_description):
        """Get code from a task description by querying a directory structure.

        This function gathers the project info using a CodebaseAgent, then asks
        a GPTAgent for the code. The function also handles JSONDecodeError and
        general exceptions, logging the errors when they occur.

        Args:
            task_description (str): The task description to get code for.

        Returns:
            code_content (str): The requested code content, or 'No code found' if unavailable.
        """
        try:
            # Gather project info
            project_structure = self.codebase_agent.get_directory_structure()

            # Formulate query for GPTAgent
            query = f'Given the project structure {project_structure}, {task_description}.'
            response_content_str = self.gpt_agent.ask_query(query)
            logging.info(f'Raw Response: {response_content_str}')

            # Deserialize the response
            response_content = json.loads(response_content_str)

            # Retrieve code from response
            code_content = response_content.get('code', 'No code found')
            return code_content

        except json.JSONDecodeError as e:
            logging.error(f'JSON Decode Error: {e}')
            logging.error(f'Failed Task Description: {task_description}')
        except Exception as e:
            logging.error(f'An unexpected error occurred: {e}')


# general imports
from pathlib import Path
import os
import re
import argparse
from time import time
import multiprocessing as mp
from functools import partial
from collections import Counter

# processing imports
import numpy as np
import pandas as pd
from tqdm import tqdm
from collections import OrderedDict
from difflib import SequenceMatcher
import os

# pdfminer imports
from pdfminer.pdfdocument import PDFDocument
from pdfminer.pdfparser import PDFParser
from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
from pdfminer.pdfpage import PDFPage
from pdfminer.converter import PDFPageAggregator
from pdfminer.layout import LTPage, LTChar, LTAnno, LAParams, LTTextBox, LTTextLine

# local imports

import rse_watch.sententizer as sententizer

def get_list_of_pdfs_filenames(dirName):
    """
        For the given path, get the List of all files in the directory tree
    """
    paths = []
    for path, subdirs, files in os.walk(dirName):
        for name in files:
            if (name.lower().endswith(".pdf")):
                paths.append((Path(path + "/" + name)))
    return paths

def get_companies_metadata_dict(config):
    """ Read companies metadata from config and turn it into dictionnary"""
    companies_metadata_dict = pd.read_csv(config.annotations_file,
                                          sep=";",
                                          encoding='utf-8-sig').set_index("project_denomination").T.to_dict()
    return companies_metadata_dict

def clean_child_str(child_str):
    child_str = ' '.join(child_str.split()).strip()
    # dealing with hyphens:
    # 1. Replace words separators in row by a different char than hyphen (i.e. longer hyphen)
    child_str = re.sub("[A-Za-z] - [A-Za-z]", lambda x: x.group(0).replace(' - ', ' – '), child_str)
    # 2. Attach the negative term to the following number, # TODO: inutile ? Enlever ?
    child_str = re.sub("(- )([0-9])", r"-\2", child_str)
    return child_str

class PDFPageDetailedAggregator(PDFPageAggregator):
    """
    Custom class to parse pdf and keep position of parsed text lines.
    """

    def __init__(self, rsrcmgr, pageno=1, laparams=None):
        PDFPageAggregator.__init__(self, rsrcmgr, pageno=pageno, laparams=laparams)
        self.rows = []
        self.page_number = 0
        self.result = ""

    def receive_layout(self, ltpage):
        def render(item, page_number):
            if isinstance(item, LTPage) or isinstance(item, LTTextBox):
                for child in item:
                    render(child, page_number)
            elif isinstance(item, LTTextLine):
                child_str = ''
                for child in item:
                    if isinstance(child, (LTChar, LTAnno)):
                        child_str += child.get_text()
                child_str = clean_child_str(child_str)
                if child_str:
                    # bbox == (pagenb, x1, y1, x2, y2, text)
                    row = (page_number, item.bbox[0], item.bbox[1], item.bbox[2], item.bbox[3], child_str)
                    self.rows.append(row)
                for child in item:
                    render(child, page_number)
            return

        render(ltpage, self.page_number)
        self.page_number += 1
        self.rows = sorted(self.rows, key=lambda x: (x[0], -x[2]))
        self.result = ltpage

def get_raw_content_from_pdf(input_file, rse_range=None):
    """
    Parse pdf file,  within rse range of pages if needed, and return list of rows with all metadata
    :param input_file: PDF filename
    :param rse_range: (nb_first_page_rse:int, nb_last_page_rse:int) tuple, starting at 1
    :return: list of rows with (pagenb, x1, y1, x2, y2, text) and page_nb starts at 0!
    """
    assert input_file.name.endswith(".pdf")
    fp = open(input_file, 'rb')

    parser = PDFParser(fp)
    doc = PDFDocument(parser)
    rsrcmgr = PDFResourceManager()
    laparams = LAParams()
    device = PDFPageDetailedAggregator(rsrcmgr, laparams=laparams)
    interpreter = PDFPageInterpreter(rsrcmgr, device)

    if rse_range is not None and rse_range != "":
        # start at zero to match real index of pages
        pages_selection = range(rse_range[0] - 1, (rse_range[1] - 1) + 1)
    else:
        pages_selection = range(0, 10000)
    first_page_nb = pages_selection[0] + 1  # to start indexation at 1
    # Checked: only useful pages are actually parsed.
    for nb_page_parsed, page in enumerate(PDFPage.create_pages(doc)):
        if nb_page_parsed in pages_selection:
            interpreter.process_page(page)
            device.get_result()

    return device, first_page_nb

def clean_paragraph(p):
    """ Curate paragraph object before save, in particular deal with hyphen and spaces """
    # Attach together words (>= 2 char to avoid things like A minus, B minus...)
    # that may have been split at end of row like géographie = "géo - graphie"
    # real separator have been turned into longer hyphen during parsing to avoid confusion with those.
    # Accents accepted thks to https://stackoverflow.com/a/24676780/8086033
    w_expr = "(?i)(?:(?![×Þß÷þø])[-'a-zÀ-ÿ]){2,}"
    p["paragraph"] = re.sub("{} - {}".format(w_expr, w_expr),
                            lambda x: x.group(0).replace(' - ', ''),
                            p["paragraph"])
    # reattach words that were split, like Fort-Cros = "Fort- Cros"
    p["paragraph"] = re.sub("{}- {}".format(w_expr, w_expr),
                            lambda x: x.group(0).replace('- ', '-'),
                            p["paragraph"])
    return p

def get_paragraphs_from_raw_content(device, idx_first_page):
    """
    From parsed data with positional information, aggregate into paragraphs using simple rationale
    :param device:
    :param idx_first_page:
    :param p: size of next gap needs to be smaller than previous min size of letters (among two last rows) times p
    :return:
    """
    # GROUPING BY COLUMN
    column_text_dict = OrderedDict()  # keep order of identification in the document.

    APPROXIMATION_FACTOR = 10  # to allow for slight shifts at beg of aligned text
    N_MOST_COMMON = 4  # e.g. nb max of columns of text that can be considered
    LEFT_SECURITY_SHIFT = 20  # to include way more shifted text of previous column
    counter = Counter()
    item_holder = []

    item_index = 0
    it_was_last_item = False
    while "There are unchecked items in device.rows":

        # add the item to the list of the page
        try:
            (page_id, x_min, _, _, _, _) = device.rows[item_index]
        except e:
            print("Wrong index {} for device.rows of len {}".format(item_index, len(device.rows)))
            print("was that last page ? : {]".format(it_was_last_item))
            raise
        item_holder.append(device.rows[item_index])

        # increment the count of x_min
        counter[(x_min // APPROXIMATION_FACTOR) * APPROXIMATION_FACTOR] += 1

        # go to next item
        it_was_last_item = item_index == (len(device.rows) - 1)
        if not it_was_last_item:
            item_index += 1
            (next_page_id, _, _, _, _, _) = device.rows[item_index]
            changing_page = (next_page_id > page_id)

        if changing_page or it_was_last_item:  # approximate next page

            top_n_x_min_approx = counter.most_common(N_MOST_COMMON)
            df = pd.DataFrame(top_n_x_min_approx, columns=["x_min_approx", "freq"])
            df = df[df["freq"] > df["freq"].sum() * (1 / (N_MOST_COMMON + 1))].sort_values(by="x_min_approx")
            x_min_approx = (df["x_min_approx"] - LEFT_SECURITY_SHIFT).values
            x_min_approx = x_min_approx * (x_min_approx > 0)
            left_x_min_suport = np.hstack([x_min_approx,
                                           [10000]])

            def x_grouper(x_min):
                delta = left_x_min_suport - x_min
                x_group = left_x_min_suport[np.argmin(delta < 0) * 1 - 1]
                return x_group

            # iter on x_group and add items
            page_nb = idx_first_page + page_id
            column_text_dict[page_nb] = {}

            for item in item_holder:
                (page_id, x_min, y_min, x_max, y_max, text) = item
                page_nb = idx_first_page + page_id
                x_group = x_grouper(x_min)
                if x_group in column_text_dict[page_nb].keys():
                    column_text_dict[page_nb][x_group].append((y_min, y_max, text))
                else:
                    column_text_dict[page_nb][x_group] = [(y_min, y_max, text)]

            if it_was_last_item:
                break
            else:
                # restart from zero for next page
                counter = Counter()
                item_holder = []

    # CREATE THE PARAGRAPHS IN EACH COLUMN
    # define minimal conditions to define a change of paragraph:
    # Being spaced by more than the size of each line (min if different to account for titles)
    pararaphs_list = []
    paragraph_index = 0
    for page_nb, x_groups_dict in column_text_dict.items():
        for x_group_name, x_groups_data in x_groups_dict.items():
            x_groups_data = sorted(x_groups_data, key=lambda x: x[0],
                                   reverse=True)  # sort vertically, higher y = before
            x_groups_data_paragraphs = []

            p = {"y_min": x_groups_data[0][0],
                 "y_max": x_groups_data[0][1],
                 "paragraph": x_groups_data[0][2]}
            previous_height = p["y_max"] - p["y_min"]
            previous_y_min = p["y_min"]
            for y_min, y_max, paragraph in x_groups_data[1:]:
                current_height = y_max - y_min
                current_y_min = y_min
                max_height = max(previous_height, current_height)

                relative_var_in_height = (current_height - previous_height) / float(
                    current_height)  # Was min before ???
                relative_var_in_y_min = abs(current_y_min - previous_y_min) / float(current_height)

                positive_change_in_font_size = (relative_var_in_height > 0.05)
                change_in_font_size = abs(relative_var_in_height) > 0.05
                different_row = (relative_var_in_y_min > 0.7)
                large_gap = (relative_var_in_y_min > 1.2)
                artefact_to_ignore = (len(paragraph) <= 2)  # single "P" broke row parsing in auchan dpef
                if not artefact_to_ignore:
                    if (positive_change_in_font_size and different_row) or large_gap:  # always break
                        # break paragraph, start new one
                        # print("break",relative_var_in_height, relative_var_in_y_min, paragraph)
                        p = clean_paragraph(p)
                        x_groups_data_paragraphs.append(p)
                        p = {"y_min": y_min,
                             "y_max": y_max,
                             "paragraph": paragraph}
                    else:
                        # if change_in_font_size:  # to separate titles
                        #     paragraph = paragraph + ".\n"
                        # paragraph continues
                        p["y_min"] = y_min
                        p["paragraph"] = p["paragraph"] + " " + paragraph

                    previous_height = current_height
                    previous_y_min = current_y_min
            # add the last paragraph of column
            p = clean_paragraph(p)
            x_groups_data_paragraphs.append(p)
            # structure the output
            for p in x_groups_data_paragraphs:
                pararaphs_list.append({"paragraph_id": paragraph_index,
                                       "page_nb": page_nb,
                                       "x_group": x_group_name,
                                       "y_min_paragraph": round(p["y_min"]),
                                       "y_max_paragraph": round(p["y_max"]),
                                       "paragraph": p["paragraph"]})
                paragraph_index += 1
    df_par = pd.DataFrame(data=pararaphs_list,
                          columns=["paragraph_id",
                                   "page_nb",
                                   "paragraph",
                                   "x_group",
                                   "y_min_paragraph",
                                   "y_max_paragraph"])
    return df_par

def parse_paragraphs_from_pdf(input_file, rse_ranges=None):
    """
    From filename, parse pdf and output structured paragraphs with filter on rse_ranges uif present.
    :param input_file: filename ending  with ".pdf" or ".PDF".
    :param rse_ranges: "(start, end)|(start, end)"
    :return: df[[page_nb, page_text]] dataframe
    """
    rse_ranges_list = list(map(eval, rse_ranges.split("|")))
    df_paragraphs_list = []
    for rse_range in rse_ranges_list:
        df_par, idx_first_page = get_raw_content_from_pdf(input_file, rse_range=rse_range)
        df_par = get_paragraphs_from_raw_content(df_par, idx_first_page)
        df_paragraphs_list.append(df_par)
    df_par = pd.concat(df_paragraphs_list, axis=0, ignore_index=True)
    return df_par

def compute_string_similarity(a, b):
    """Compares two strings and returns a similarity ratio between 0 and 1 """
    return SequenceMatcher(None, a, b).ratio()

def cut_footer(df_par, p=0.8, verbose=False):
    """
    Cut the paragraph with lowest y_min if other paragraphs are similar.
    The similarity is measured with function compute_string_similarity
    """
    len_first = len(df_par)
    footers = []
    deno = df_par['project_denomination'].values[0]
    c = 0
    while True:
        c += 1
        len_start = len(df_par)
        y_bottom = df_par['y_min_paragraph'].min()
        y_top = df_par[df_par['y_min_paragraph'] == y_bottom]['y_max_paragraph'].min()
        DSmin = df_par[(df_par['y_max_paragraph'] == y_top) & (df_par['y_min_paragraph'] == y_bottom)].copy()
        if len(DSmin) == 1 and c == 1:
            if verbose:
                print('\n', deno)
            return df_par
        if len(DSmin) == 1:
            break
        for candidate in DSmin['paragraph'].values:
            DSmin['is_foot'] = DSmin['paragraph'].apply(lambda x: compute_string_similarity(str(x), candidate) > p)
            count = len((DSmin[DSmin['is_foot'] == True]))
            if count > 1:
                footers.append((candidate, count))
                index_foot = DSmin[DSmin['is_foot'] == True].index
                break
            else:
                DSmin = DSmin.drop(DSmin.index[0])
        if len(footers) == 0:
            if verbose:
                print('\n', deno)
            return df_par
        len_end = (len(df_par[~df_par.index.isin(index_foot)]))
        df_par = df_par[~df_par.index.isin(index_foot)]
        if len_start == len_end:
            break
    # Below part is for human check that the function works properly
    # if verbose:
    #     len_last = len(df_par)
    #     S = sum([i for _,i in footers])
    #     print('\n',deno)
    #     print(f"Removed {len_first-len_last} lines. {len_first-len_last==S}")
    #     if footers!=[]:
    #         L = [foot+" x "+ str(count) for foot, count in footers]
    #         print("Footers(s) --->\n",'\n '.join(L))
    return df_par

def cut_header(df_par, p=0.8, verbose=False):
    "Same as function cut_footer() but for headers"
    len_first = len(df_par)
    headers = []
    deno = df_par['project_denomination'].values[0]
    c = 0
    while True:
        c += 1
        len_start = len(df_par)
        y_top = df_par['y_max_paragraph'].max()
        y_bottom = df_par[df_par['y_max_paragraph'] == y_top]['y_min_paragraph'].max()
        DSmax = df_par[(df_par['y_max_paragraph'] == y_top) & (df_par['y_min_paragraph'] == y_bottom)].copy()
        if len(DSmax) == 1 and c == 1:
            if verbose:
                print('\n', deno)
            return df_par
        if len(DSmax) == 1:
            break
        for candidate in DSmax['paragraph'].values:
            DSmax['is_head'] = DSmax['paragraph'].apply(lambda x: compute_string_similarity(str(x), candidate) > p)
            count = len((DSmax[DSmax['is_head'] == True]))
            if count > 1:
                headers.append((candidate, count))
                index_head = DSmax[DSmax['is_head'] == True].index
                break
            else:
                DSmax = DSmax.drop(DSmax.index[0])
        if len(headers) == 0:
            if verbose:
                print('\n', deno)
            return df_par
        len_end = (len(df_par[~df_par.index.isin(index_head)]))
        df_par = df_par[~df_par.index.isin(index_head)]
        if len_start == len_end:
            break

    # Below part is for human check that the function works properly
    # if verbose:
    #     len_last = len(df_par)
    #     S = sum([i for _, i in headers])
    #     print('\n', deno)
    #     print(f"Removed {len_first - len_last} lines. {len_first - len_last == S}")
    #     if headers != []:
    #         L = [head + " x " + str(count) for head, count in headers]
    #         print("Header(s) --->\n", '\n '.join(L))

    return df_par

def extract_company_metadata(dpef_path, companies_metadata_dict):
    """ From metadata dict and a dpef file, get the relevant info."""
    project_denomination = dpef_path.name.split("_")[0]
    document_year = dpef_path.name.split("_")[1]
    company_name = companies_metadata_dict[project_denomination]["denomination"]
    company_sectors = None
    try:
        company_sectors = companies_metadata_dict[project_denomination]["sectors"].split(";")
    except KeyError:
        print("Sectors not found for company {} for year {}".format(project_denomination,
                                                                    document_year))
    rse_ranges = None
    try:
        rse_ranges = companies_metadata_dict[project_denomination]["rse_ranges_" + document_year]
    except KeyError:
        print("RSE ranges not found for company {} for year {}".format(project_denomination,
                                                                       document_year))

    return company_name, project_denomination, company_sectors, document_year, rse_ranges

def get_paragraphs_dataframe_from_pdf(dpef_path, companies_metadata_dict):
    """
    Parse a pdf and return a pandas df with paragraph level parsed text.

    :param dpef_path_dict_annotations: (inpout_file, dict_annotations) tuple
    :return:
    """

    company_name, project_denomination, company_sectors, document_year, rse_ranges \
        = extract_company_metadata(dpef_path, companies_metadata_dict)
    t = time()
    print("Start parsing -  {} [{}]".format(
        project_denomination,
        dpef_path.name)
    )
    df_par = parse_paragraphs_from_pdf(dpef_path, rse_ranges=rse_ranges)
    df_par.insert(0, "project_denomination", project_denomination)
    df_par.insert(1, "company_sector", ";".join(company_sectors))
    df_par.insert(2, "document_year", document_year)
    df_par = df_par.drop_duplicates(['paragraph'])
    df_par = cut_footer(df_par, verbose=True)
    df_par = cut_header(df_par, verbose=True)

    print("End parsing - {} [{}] - took {} seconds".format(
        project_denomination,
        dpef_path.name,
        int(t - time()))
    )
    return df_par

def get_sentences_dataframe_from_pdf(config, dpef_path):
    """ Parse a pdf and return a pandas df with sentence level parsed text"""
    companies_metadata_dict = get_companies_metadata_dict(config)
    df_par = get_paragraphs_dataframe_from_pdf(dpef_path, companies_metadata_dict)
    df_sent = sententizer.get_sentence_dataframe_from_paragraph_dataframe(df_par, config)
    return df_sent

def get_sentences_from_all_pdfs(config):
    """
    Parses all dpefs into a sentence-level format and save the resulting csv according to config.
    """
    companies_metadata_dict = get_companies_metadata_dict(config)
    all_dpef_paths = get_list_of_pdfs_filenames(config.dpef_dir)
    all_dpef_paths = [dpef_path for dpef_path in all_dpef_paths if
                       dpef_path.name.split("_")[0] in companies_metadata_dict.keys()]
    print(all_dpef_paths)

    # PARALLELIZATION
    parallel_get_sentences_dataframe_from_pdf = partial(get_sentences_dataframe_from_pdf,
                                                        config)
    n_cores = mp.cpu_count() - 1 or 1

    with mp.Pool(n_cores) as pool:
        print("Multiprocessing with {} cores".format(n_cores))
        df_sents = list(
            tqdm(
                pool.imap(
                    parallel_get_sentences_dataframe_from_pdf,
                    all_dpef_paths
                ),
                total=len(all_dpef_paths)
            )
        )

    # concat
    df_sents = pd.concat(df_sents, axis=0, ignore_index=True)
    # create parent folder
    pickle_path = config.parsed_sent_file.parent
    pickle_path.mkdir(parents=True, exist_ok=True)
    # save to csv
    df_sents.to_csv(config.parsed_sent_file, sep=";", index=False, encoding='utf-8-sig')

    return df_sents

def run(config):
    """
    Parse the pdfs into structured csv formats
    : param conf: conf object with relative paths.
    :param task: "parser", "sententizer" or "both" ; Whether to parse
    pdfs, sententize the paragraphs, or do both.
    """
    df_sents = get_sentences_from_all_pdfs(config)
    print(df_sents.shape)
    return df_sents


from flask import Blueprint, request
from flask import jsonify, render_template
from authlib.specs.rfc6749 import OAuth2Error
from authlib.flask.oauth2 import current_token
from ..models import OAuth2Client, OAuth2Token, User
from ..auth import current_user
from ..forms.auth import ConfirmForm, LoginConfirmForm
from ..services.oauth2 import authorization, scopes, require_oauth
from urlparse import parse_qs

from flask_cors import CORS

bp = Blueprint('oauth2', __name__)
CORS(bp)

@bp.route('/authorize', methods=['GET', 'POST'])
def authorize():
    curr_url = '/oauth2/authorize?' + request.query_string
    ##app.logger.info("method oauth2/authorize")

    if current_user:
        ##app.logger.info("confirm form")
        form = ConfirmForm()
    else:
        ##app.logger.info("login confirm form")
        form = LoginConfirmForm()

    if form.validate_on_submit():
        ##app.logger.info("submit")
        if form.confirm.data:
            # granted by current user
            grant_user = current_user
        else:
            grant_user = None
        ##app.logger.info("calling lib function 'create_authorization_response'")
        ret = authorization.create_authorization_response(grant_user)
        ##app.logger.info("return")
        return ret
    try:
        ##app.logger.info("not submit")
        ##app.logger.info("calling lib function 'validate_authorization_request'")
        grant = authorization.validate_authorization_request()
    except OAuth2Error as error:
        # TODO: add an error page
        payload = dict(error.get_body())
        return jsonify(payload), error.status_code

    client = OAuth2Client.get_by_client_id(request.args['client_id'])
    ##app.logger.info("render")
    return render_template(
        'account/authorize.html',
        grant=grant,
        scopes=scopes,
        client=client,
        form=form,
    )

@bp.route('/token', methods=['POST'])
def issue_token():
    ##app.logger.info("method oauth2/token")
    ##app.logger.info("calling lib function 'create_token_response'")
    ret = authorization.create_token_response()
    ##app.logger.info("return")
    return ret

@bp.route('/revoke', methods=['POST'])
def revoke_token():
    return authorization.create_revocation_response()

@bp.route('/revoke_bearer', methods=['POST'])
def revoke_token_bearer():
    ##app.logger.info("method oauth2/revoke_bearer")
    ##app.logger.info("query token")
    token = OAuth2Token.query_token(parse_qs(request.query_string)['token'][0])
    if token:
        ##app.logger.info("revoke")
        token.revoke()
        ##app.logger.info("return")
        return jsonify(token)
    return jsonify({'error': 'Invalid token supplied'}), 401

@bp.route('/tokeninfo', methods=['GET'])
def get_token_info():
    ##app.logger.info("method oauth2/tokeninfo")
    if 'access_token' in request.args:
        ##app.logger.info("query token")
        token = OAuth2Token.query_token(request.args['access_token'])
        if token and token.user_id:
            ##app.logger.info("query user")
            user = User.query.get(token.user_id)
            udict = user.to_dict(request.host)
            udict.update(token.to_dict())
            ##app.logger.info("return user info")
            return jsonify(udict)
        return jsonify({'error': 'Invalid token supplied'}), 401
    return jsonify({'error': 'Invalid parameters supplied'}), 400

@bp.route('/emailinfo', methods=['GET'])
def get_email_info():
    ##app.logger.info("method oauth2/emailinfo")
    if 'email' in request.args and 'access_token' in request.args:
        ##app.logger.info("query token")
        token = OAuth2Token.query_token(request.args['access_token'])
        email = request.args['email']
        if token and token.user_id:
            ##app.logger.info("query user")
            user = User.query_email(email)
            if user:
                udict = user.to_dict(request.host)
                ##app.logger.info("return user info")
                return jsonify(udict)
            return jsonify({'error': 'Invalid email supplied'}), 404
        return jsonify({'error': 'Invalid token supplied'}), 401
    return jsonify({'error': 'Invalid parameters supplied'}), 400


import requests
import pprint
import matplotlib.pyplot as plt
from tabulate import tabulate
import numpy as np

print('')
print('- GLOBAL INFO -')
print('')

# CONNECT WITH THE API

url = 'https://api.coingecko.com/api/v3/coins/categories'

r = requests.get(url)
response = r.json()

# pprint.pprint(response)
# exit()

# GET TOTAL MARKET CAP

url1 = 'https://api.coingecko.com/api/v3/global'

r1 = requests.get(url1)
response1 = r1.json()

total_marketcap = round(response1['data']['total_market_cap']['usd'], 3)
total_volume = round(response1['data']['total_volume']['usd'], 3)
print('Total Crypto MarketCap: {0:12,.3f}'.format(total_marketcap))
print('Total Crypto 24h Volume: {0:12,.3f}'.format(total_volume))

bb = round(total_volume / total_marketcap * 100, 2)
print('Total Volume 24h / Total MarketCap (%): ', bb ,'%')

print('')
# GET THE NAME OF THE SECTORES

sector_l = list()
id_l = list()
marketcap_l = list()
data_for_table_l = list()
market_share_perc_l = list()
volume_24h_l = list()
volume_24h_perc_l = list()
perc_vol_marketcap_l = list()
market_cap_changes_24h_l = list()
n_l = list()
positive_marketcap_changes = list()
negative_marketcap_changes = list()
for i in range(len(response)):
    v = response[i]['name']
    id = response[i]['id']
    n = i + 1

    try:
        market_cap = round(float(response[i]['market_cap']), 3)
        market_cap_changes_24h = round(float(response[i]['market_cap_change_24h']), 3)
    except:
        market_cap = 0
        market_cap_changes_24h = 0
        continue
    perc = round(market_cap / total_marketcap * 100, 2)
    try:
        volume_24h = round(float(response[i]['volume_24h']), 3)
    except:
        volume_24h = 0
    perc_volume = round(volume_24h / total_volume * 100, 2)
    if market_cap != 0:
        perc_vol_marketcap = round(volume_24h / market_cap * 100, 2)
    else:
        perc_vol_marketcap = 0

    # INSERT INTO LISTS

    sector_l.append(v)
    marketcap_l.append(market_cap)
    market_share_perc_l.append(perc)
    volume_24h_l.append(volume_24h)
    volume_24h_perc_l.append(perc_volume)
    perc_vol_marketcap_l.append(perc_vol_marketcap)
    id_l.append(id)
    market_cap_changes_24h_l.append(market_cap_changes_24h)
    n_l.append(n)
    if market_cap_changes_24h > 0:
        positive_marketcap_changes.append(market_cap_changes_24h)
    elif market_cap_changes_24h <0:
        negative_marketcap_changes.append(market_cap_changes_24h)

    # CALCULATIONS

    if market_cap / 1000000 < 1000:
        marketcap2 = str(round(market_cap / 1000000, 3)) + ' M'
    else:
        marketcap2 = str(round(market_cap/1000000000, 3)) + ' B'

    if volume_24h / 1000000 < 1000:
        volume_24h2 = str(round(volume_24h / 1000000, 3)) + ' M'
    else:
        volume_24h2 = str(round(volume_24h / 1000000000, 3)) + ' B'

    a = (n, v, id, marketcap2, perc, volume_24h2, perc_volume, perc_vol_marketcap, market_cap_changes_24h)
    data_for_table_l.append(a)

# PRINT MORE VALUES

avg_marketcap_perc = round(np.average(market_cap_changes_24h_l), 3)
print('Avg MCap 24h %: ', avg_marketcap_perc, '%')
positive_avg_marketcap_perc = round(np.average(positive_marketcap_changes), 3)
print('Positive sectors - Avg MCap 24h %: ', positive_avg_marketcap_perc, '%')
negative_avg_marketcap_perc = round(np.average(negative_marketcap_changes), 3)
print('Negative sectors - Avg MCap 24h %: ', negative_avg_marketcap_perc, '%')
print('')

# PRINT VALUES IN TABLE

a = input('You want to see the values in table (yes/no): ')
if a == 'yes':
    head = ['n',
            'Sector',
            'id',
            'MarketCap',
            'MCap-TotMCap %',
            'Volume 24h',
            'Vol-TotVol %',
            'Vol-MCap %',
            'MCap 24h %'
            ]
    print(tabulate(data_for_table_l, headers=head, tablefmt='grid'))

# q1 = input('Do you wanna order by the MCap 24h % (yes/no): ')
# if q1 == 'yes':
#     sorted_list = sorted(data_for_table_l, key=lambda x: x[8], reverse=True)
#     head = ['n',
#             'Sector',
#             'id',
#             'MarketCap',
#             'MCap-TotMCap %',
#             'Volume 24h',
#             'Vol-TotVol %',
#             'Vol-MCap %',
#             'MCap 24h %'
#             ]
#     print(tabulate(sorted_list, headers=head, tablefmt='grid'))
#
# # CREATE GRAPH TO GET ALL INFO
#
# print('')
# b1 = input('Graphs (yes/no): ')
# if b1 == 'yes':
#     b = input('Graph by market cap (yes/no): ')
#     if b == 'yes':
#         fig, (ax1, ax2) = plt.subplots(2, sharex=True)
#         fig.suptitle ('All crypto sectores')
#
#         ax1.bar(sector_l, marketcap_l)
#         ax2.bar(sector_l, marketcap_l)
#
#         plt.yscale('log')
#         plt.xticks(fontsize=8, rotation='vertical')
#         plt.ylabel('MarketCap')
#
#
#         plt.tight_layout()
#         plt.subplots_adjust(hspace=0.05)
#         plt.show()
#
#     c = input('Graph Market share (yes/no): ')
#     if c =='yes':
#         plt.bar(sector_l, market_share_perc_l)
#         plt.title('Market share by sectores')
#         plt.ylabel('%')
#         plt.xticks(fontsize=8, rotation='vertical')
#
#         plt.tight_layout()
#         plt.show()
#
#     d = input('Graph for MarketCap and Volume (yes/no): ')
#     if d == 'yes':
#         fig, (ax1, ax2, ax3) = plt.subplots(3, sharex=True)
#         fig.suptitle('Info for all sectores')
#
#         size = np.arange(len(sector_l))
#
#         bar1 = ax1.bar(size, marketcap_l, 0.35, label='MarketCap')
#         bar2 = ax1.bar(size + 0.25, volume_24h_l, 0.35,  label='Volume 24h')
#
#         ax1.legend((bar1, bar2), ('MarketCap', 'Volume 24h'))
#
#         bar3 = ax2.bar(size, market_share_perc_l, 0.35, label = 'Market share %')
#         bar4 = ax2.bar(size + 0.5, volume_24h_perc_l, 0.35, label='Vol/Total Volume %')
#         bar5 = ax2.bar(size + 0.25, perc_vol_marketcap_l, 0.35, label = 'Vol/MarketCap %')
#
#         ax2.legend((bar3, bar4, bar5), ('Market share %', 'Vol/Total Volume %', 'Vol/MarketCap %'))
#
#         bar6 = ax3.bar(size, market_share_perc_l, 0.35, label = 'Market share %')
#         bar7 = ax3.bar(size + 0.5, volume_24h_perc_l, 0.35, label='Vol/Total Volume %')
#         bar8 = ax3.bar(size + 0.25, perc_vol_marketcap_l, 0.35, label = 'Vol/MarketCap %')
#
#         ax3.legend((bar6, bar7, bar8), ('Market share %', 'Vol/Total Volume %', 'Vol/MarketCap %'), prop={'size':6})
#
#         plt.yscale('log')
#
#         plt.xticks(size + 0.25, sector_l, fontsize=8, rotation='vertical')
#         plt.subplots_adjust(hspace=0.05)
#         plt.show()

print('\n-- DATABASE INFORMATION \n')
import sqlite3

# Create a connection to the SQLite database and create a cursor object
conn = sqlite3.connect("crypto_data.db")
cursor = conn.cursor()

# Get the current date
import datetime
current_date = datetime.datetime.now().strftime("%Y_%m_%d")

# Function to check if a column exists in a table
def column_exists(cursor, table_name, column_name):
    cursor.execute(f"PRAGMA table_info({table_name})")
    columns = [column[1] for column in cursor.fetchall()]
    return column_name in columns

# Create market_cap table if it doesn't exist
cursor.execute("""
CREATE TABLE IF NOT EXISTS market_cap (
    sector_id TEXT PRIMARY KEY,
    sector_name TEXT
)
""")
print("Created market_cap table if not exists")

# Create daily_volume table if it doesn't exist
cursor.execute("""
CREATE TABLE IF NOT EXISTS daily_volume (
    sector_id TEXT PRIMARY KEY,
    sector_name TEXT
)
""")
print("Created daily_volume table if not exists")

# Add a prefix to the current_date to avoid column names starting with a number
current_date_column = f"date_{current_date}"

# Add a column for the current date if it doesn't exist
if not column_exists(cursor, "market_cap", current_date_column):
    cursor.execute(f"ALTER TABLE market_cap ADD COLUMN {current_date_column} REAL")
    print(f"Added column {current_date_column} to market_cap table")

if not column_exists(cursor, "daily_volume", current_date_column):
    cursor.execute(f"ALTER TABLE daily_volume ADD COLUMN {current_date_column} REAL")
    print(f"Added column {current_date_column} to daily_volume table")

# Update market cap and daily volume information for each sector
for i in range(len(response)):
    sector_id = response[i]['id']
    sector_name = response[i]['name']

    try:
        market_cap = round(float(response[i]['market_cap']), 3)
    except:
        market_cap = None

    try:
        volume_24h = round(float(response[i]['volume_24h']), 3)
    except:
        volume_24h = None

    print(f"Processing sector: {sector_name}")

    # Insert or update the sector in the market_cap table
    cursor.execute("""
    INSERT OR IGNORE INTO market_cap (sector_id, sector_name)
    VALUES (?, ?)
    """, (sector_id, sector_name))
    if market_cap is not None:
        cursor.execute(f"""
        UPDATE market_cap
        SET {current_date_column} = ?
        WHERE sector_id = ?
        """, (market_cap, sector_id))
        print(f"Updated market cap for {sector_name} on {current_date}")

    # Insert or update the sector in the daily_volume table
    cursor.execute("""
    INSERT OR IGNORE INTO daily_volume (sector_id, sector_name)
    VALUES (?, ?)
    """, (sector_id, sector_name))
    if volume_24h is not None:
        cursor.execute(f"""
        UPDATE daily_volume
        SET {current_date_column} = ?
        WHERE sector_id = ?
        """, (volume_24h, sector_id))
        print(f"Updated daily volume for {sector_name} on {current_date}")

# Commit the changes and close the connection
conn.commit()
conn.close()
print("\n Changes committed and connection closed")


import numpy as np
import matplotlib.pyplot as plt

from starwhale import dataset

def show_image(image) -> None:
    plt.imshow(image, cmap="gray")
    plt.show(block=True)

ds_name = "fer2013/version/latest"
ds = dataset(ds_name)
row = ds.fetch_one()
data = row.features
show_image(
    np.frombuffer(data["image"].to_bytes(), dtype=np.uint8).reshape(data["image"].shape)
)
print(data["label"])


# D = len(data) ## number of docs...data is list of documents
# print('D: ', D)

import torch
import pickle

beta = torch.load('betas-40.pt', map_location=torch.device('cpu'))
beta = beta.numpy()

with open('vocab_50K.pkl', 'rb') as f:
    vocab = pickle.load(f)

TC = []
num_topics = len(beta)
for k in range(num_topics):
    print('k: {}/{}'.format(k, num_topics))
    top_10 = list(beta[k].argsort()[-11:][::-1])
    top_words = [vocab[a] for a in top_10]
    TC_k = 0
    counter = 0