from importlib.resources import contents
import logging
import json

from aiogram import Bot, Dispatcher, types
from aiogram.types import Message
from data.config import BOT_TOKEN

from aiogram import types
# from aiogram.dispatcher.filters.builtin import CommandStart
from aiogram.types import InlineKeyboardMarkup, InlineKeyboardButton
from aiogram.types.web_app_info import WebAppInfo
from aiohttp.web_app import Application

#
from aiogram.dispatcher.webhook.aiohttp_server import SimpleRequestHandler, setup_application
from aiogram.utils.web_app import safe_parse_webapp_init_data
from aiohttp.web_request import Request
from aiohttp.web_response import json_response
from aiogram.dispatcher.router import Router
from aiogram.dispatcher.filters.content_types import ContentTypesFilter, ContentType

from aiogram import Bot
from aiogram.types import (
    InlineKeyboardButton,
    InlineKeyboardMarkup,
    InlineQueryResultArticle,
    InputTextMessageContent,
    ReplyKeyboardMarkup,
    KeyboardButton,
    WebAppInfo,
)
from aiogram.utils.web_app import check_webapp_signature, safe_parse_webapp_init_data
from aiogram.types import MenuButtonWebApp, WebAppInfo
from aiohttp.web import run_app

from routes import check_data_handler, send_message_handler
from aiogram.dispatcher.filters import Command

TOKEN = BOT_TOKEN
bot = Bot(token=TOKEN, parse_mode="HTML")
dp = Dispatcher()
my_router = Router()

logger = logging.getLogger(__name__)

@my_router.message(Command(commands=["start"]))
async def command_start_handler(message: types.Message):

    keyboard = InlineKeyboardMarkup(
        inline_keyboard= [[
            InlineKeyboardButton(text="Google", web_app=WebAppInfo(url="https://cumpbotweb.dobbikov.com/"))
        ]]
    )
    reply_keyboard = ReplyKeyboardMarkup(
        keyboard = [[
            KeyboardButton(text = "Подати заявку", web_app = WebAppInfo(url="https://cumpbotweb.dobbikov.com/"))
        ]]
    )
    await message.answer(f"Привіт, {message.from_user.full_name}!", reply_markup=reply_keyboard)

@my_router.message(ContentTypesFilter(content_types=[ContentType.WEB_APP_DATA]))
async def web_app_handler(message: types.Message) -> None:
    # print(message.web_app_data)
    # print("remove it!!")
    print(message.web_app_data.data)
    data = json.loads(message.web_app_data.data)
    print(data["name"])
    print(data["email"])
    print(data["discord"])
    print(data["nickname"])
    print(data["age"])

    name = data["name"]
    email = data["email"]
    discord = data["discord"]
    nickname = data["nickname"]
    age = data["age"]

    await bot.send_message(
        -1001768046932,
        f"{name} надіслав заявку:\n\n\
Email: {email}\n\
Discord: {discord}\n\
Nickname: {nickname}\n\
Вік: {age}\n\
", parse_mode="HTML")
    await bot.send_message(message.from_user.id, "Дякуємо за надіслану заявку!\nМи її отримали та відповімо вам, коли її перевіримо.\nГарного дня!")

    # message.answer("{message.web_app_data.data}")

@my_router.message()
async def echo_handler(message: types.Message) -> None:
    """
    Handler will forward received message back to the sender
    By default message handler will handle all message types (like text, photo, sticker and etc.)
    """
    try:
        # Send copy of the received message
        await message.send_copy(chat_id=message.chat.id)
    except TypeError:
        # But not all the types is supported to be copied so need to handle it
        await message.answer("Nice try!")

async def on_startup(bot: Bot, base_url: str):
    # dp.register_message(command_start_handler, commands=['start'])
    # Router.register_message(command_start_handler, commands=['start'])
    webhook = await bot.get_webhook_info()

    WEBHOOK_URL = f"https://cumpbot.dobbikov.com/webhook"

    # If URL is bad
    if webhook.url != WEBHOOK_URL:
        # If URL doesnt match current - remove webhook
        if not webhook.url:
            await bot.delete_webhook()

        # Set new URL for webhook
        await bot.set_webhook(WEBHOOK_URL)
        # If you want to use free certificate signed by LetsEncrypt you need to set only URL without sending certificate.

    # await bot.set_webhook(f"{base_url}/webhook")
    # await bot.set_chat_menu_button(
    #     menu_button=MenuButtonWebApp(text="Open Menu", web_app=WebAppInfo(url=f"https://cumpbotweb.dobbikov.com/"))
    # )

def main() -> None:
    # Initialize Bot instance with an default parse mode which will be passed to all API calls
    # bot = Bot(TOKEN, parse_mode="HTML")
    # And the run events dispatching
    # dp.run_polling(bot)

    dispatcher = Dispatcher()
    dispatcher["base_url"] = f"https://cumpbot.dobbikov.com"
    dispatcher.startup.register(on_startup)

    dispatcher.include_router(my_router)

    app = Application()
    app["bot"] = bot
    # app.router.add_get("/demo", demo_handler)
    # app.router.add_post("/demo/checkData", check_data_handler)
    # app.router.add_post("/demo/sendMessage", send_message_handler)
    SimpleRequestHandler(
        dispatcher=dispatcher,
        bot=bot,
    ).register(app, path="/webhook")
    setup_application(app, dispatcher, bot=bot)

    run_app(app, host="127.0.0.1", port=3003)

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('bar', '0300_auto_20161002_1441'),
        ('clientes', '0276_auto_20160827_2010'),
        ('personal', '0272_auto_20160827_2010'),
        ('stock', '0204_productoventa'),
        ('ventas', '0050_auto_20161002_1429'),
    ]

    operations = [
        migrations.CreateModel(
            name='Pedido',
            fields=[
                ('numero_pedido', models.AutoField(help_text=b'Este dato se genera automaticamente cada vez que se va crear un Pedido.', serialize=False, verbose_name=b'Nro. Pedido', primary_key=True)),
                ('fecha_pedido', models.DateTimeField(help_text=b'La fecha y hora del Pedido se asignara automaticamente una vez que sea guardado.', verbose_name=b'Fecha/Hora del Pedido', auto_now_add=True)),
                ('total_pedido', models.DecimalField(default=0, verbose_name=b'Total del Pedido', max_digits=18, decimal_places=0)),
                ('estado_pedido', models.ForeignKey(default=1, verbose_name=b'Estado del Pedido', to='bar.PedidoEstado', help_text=b'El estado del Pedido se establece automaticamente.')),
                ('mesa_pedido', models.ManyToManyField(help_text=b'Indique la/s mesa/s que sera/n ocupada/s por el/los Cliente/s.', to='bar.Mesa', verbose_name=b'Mesas disponibles')),
                ('mozo_pedido', models.ForeignKey(verbose_name=b'Atendido por?', to_field=b'usuario', to='personal.Empleado', help_text=b'Este dato se completara automaticamente cuando el Pedido sea guardado.')),
                ('reserva', models.ForeignKey(blank=True, to='clientes.Reserva', help_text=b'Seleccione una Reserva en caso de que el Cliente haya realizado una.', null=True)),
            ],
            options={
                'verbose_name': 'Pedido',
                'verbose_name_plural': 'Pedidos',
            },
        ),
        migrations.CreateModel(
            name='PedidoDetalle',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('precio_producto_pedido', models.DecimalField(help_text=b'El Precio de Venta del Producto se define en la pantalla de Productos.', verbose_name=b'Precio Venta Producto', max_digits=18, decimal_places=0)),
                ('cantidad_producto_pedido', models.DecimalField(default=1, help_text=b'Ingrese la cantidad del producto solicitado por el Cliente.', verbose_name=b'Cantidad del Producto', max_digits=10, decimal_places=3)),
                ('total_producto_pedido', models.DecimalField(default=0, help_text=b'Este valor se calcula automaticamente tomando el Precio Venta del Producto por la Cantidad del Producto.', verbose_name=b'Costo Total del Producto', max_digits=18, decimal_places=0)),
                ('fecha_pedido_detalle', models.DateTimeField(help_text=b'Registra la fecha y hora en que se realizo el detalle del Pedido, util cuando el cliente pide mas productos.', verbose_name=b'Fecha/hora del detalle del Pedido', auto_now_add=True)),
                ('pedido', models.ForeignKey(to='ventas.Pedido')),
                ('producto_pedido', models.ForeignKey(verbose_name=b'Producto a ordenar', to='stock.ProductoVenta', help_text=b'Seleccione el Producto ordenado por el Cliente.')),
            ],
            options={
                'verbose_name': 'Pedido - Detalle',
                'verbose_name_plural': 'Pedidos - Detalles',
            },
        ),
        # migrations.AddField(
        #     model_name='comanda',
        #     name='numero_pedido',
        #     field=models.ForeignKey(default=0, to='ventas.Pedido'),
        # ),
        # migrations.AddField(
        #     model_name='venta',
        #     name='numero_pedido',
        #     field=models.OneToOneField(default=0, to='ventas.Pedido', help_text=b'Seleccione el Numero de Pedido para el cual se registrara la Venta.', verbose_name=b'Numero de Pedido'),
        # ),
    ]


from fastapi import HTTPException, Request
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
from sentry_sdk import capture_message
from ..auth.auth_handler import verify_jwt

class JWTBearer(HTTPBearer):
    def __init__(self, auto_error: bool = True):
        # automatic error reportin
        super(JWTBearer, self).__init__(auto_error=auto_error)

    async def __call__(self, request: Request):
        credentials: HTTPAuthorizationCredentials | None = await super(
            JWTBearer, self
        ).__call__(request)
        if credentials:
            if not credentials.scheme == "Bearer":
                e = HTTPException(
                    status_code=403, detail="Invalid authentication scheme."
                )
                capture_message("Invalid authentication scheme.")
                raise e
            if not verify_jwt(credentials.credentials):
                e = HTTPException(
                    status_code=403, detail="Invalid token or expired token."
                )
                capture_message("Invalid token or expired token.")
                raise e
            return credentials.credentials
        else:
            e = HTTPException(status_code=403, detail="Invalid authorization code.")
            capture_message("Invalid authorization code.")
            raise e


from django.shortcuts import render, redirect
from .models import Story, TagsModel
from .forms import StoryForm, ProductFilter
from django.http import HttpResponse
from django.views.generic import DetailView, UpdateView, DeleteView

def product_list(request):
    filter = ProductFilter(request.GET, queryset=Story.objects.all())
    return render(request, 'main/index.html', {'filter': filter})

def index(request):
    tasks = Story.objects.all()
    count = tasks.count()
    choices = TagsModel.objects.all()
    answer = ''
    kateg = ''
    if request.method == 'POST':
        answer = int(request.POST.get('filter_by'))

        #tags = "в категории " + str(TagsModel.objects.get(tagId=answer).tagName)
        if answer == 1:
            tasks = Story.objects.filter(classif = 'Сказки о животных')
        else:
            tasks = Story.objects.filter(classif = 'Волшебные сказки')
        count = tasks.count()
    # forDifferent = FilterTag()
    context = {
        'title': 'Главная страница сайта',
        'tasks': tasks,
        'choices': choices,
        'count': count,
    }
    return render(request, 'main/index.html', context)

def about(request):
    return render(request, 'main/about.html')

def create(request):
    error = ''
    if request.method == 'POST':
        form = StoryForm(request.POST)
        if form.is_valid():
            form.save()
            return redirect('home')
        else:
            error = "Форма была неверной"

    form = StoryForm()
    context = {
        'form': form
    }
    return render(request, 'main/create.html', context)

def story(request):
    tasks = Story.objects.filter(classif = 'Волшебные сказки')
    return render(request, 'main/story.html', {'tasks': tasks})

    #return render(request, 'main/story.html')

def story_2(request):

    tasks = Story.objects.filter(classif='Сказки о животных')
    return render(request, 'main/story_2.html', {'tasks': tasks})

    #return render(request, 'main/story_2.html')

from django.shortcuts import render
from django.http import HttpResponseRedirect
from .forms import ResumeForm

# Создайте здесь представления.

def upload_resume(request):
    error = ''
    if request.method == 'POST':
        form = ResumeForm(request.POST, request.FILES)
        if form.is_valid():
            form.save()
            return redirect('/resume/')
        else:
            error = "Форма была неверной"
    else:
        form = ResumeForm()
    return render(request, 'files/resume.html', {'form':form})

from .classificator import MainClassify as MC
from .forms import NewForm, TextsFormSecond

def classify(request):
    error = ''
    classText = ''
    if request.method == 'POST':
        form = TextsFormSecond(request.POST)
        secondForm = NewForm(request.POST)
        if form.is_valid():
            text = form.cleaned_data.get("text_news")
            classText = MC.choose_class(text)
        else:
            error = 'Попробуйте ввести другие данные'
    form = NewForm()
    secondForm = NewForm()
    context = {
        'form': form,
        'error': error,
        'predict': classText
    }
    return render(request, 'main/classify.html', context)

def my_test_500_view(request):
    # Return an "Internal Server Error" 500 response code.
    return HttpResponse(status=500)

def showAll(request):
    news = Story.objects.all()
    count = news.count()
    return index(request)


from discord.ext import commands
from lib.mysqlwrapper import mysql
from typing import Optional
import discord
import lib.embedder
import logging

class Checklist(commands.Cog):
    def __init__(self, client):
        self.client = client

        # Set up the logger
        self.logger = logging.getLogger(__name__)
        self.logger.addHandler(logging.NullHandler())

        self.logger.info("Loading checklist cog")

    def cog_unload(self):
        self.logger.info("Unloading checklist cog")

    @commands.group(
        name = "shiny",
        aliases = ["s"],
        brief = "Shiny checklist system",
        description = "Cherubi Bot - Shiny Checklist System",
        usage = "<add | remove | list>"
    )
    async def shiny_group(self, ctx):
        # If no subcommand is given, give the help command for the group
        if ctx.invoked_subcommand is None:
            await ctx.send_help(str(ctx.command))

    @shiny_group.command(
        name = "add",
        aliases = ["a"],
        brief = "Adds a shiny Pokémon to your list",
        description = "Cherubi Bot - Shiny Checklist System (Add)",
        usage = "<name or dex #> [number]",
        help = "You can give either the name or the dex number of the Pokémon to add it to your list.\n\nYou also can give an amount, if you don't it'll add a single one."
    )
    async def add_subcommand(self, ctx, pokemon, count = 1):
        # Check that the user has their home guild set. If not, then set it.
        # Check if this was invoked from a guild
        if not isinstance(ctx.channel, discord.DMChannel):
            db = mysql()
            query = """
                SELECT
                    user_id,
                    home_guild
                FROM user_preferences
                WHERE user_id = %s;
            """
            results = db.query(query, [ctx.author.id])
            db.close()

            # If nothing was returned, then invoke the sethome command
            if not results or not results[0]['home_guild']:
                await ctx.invoke(self.client.get_command("sethome"))

        # Just a couple of sanity checks, since I know someone will test this at some point
        if count == 0:
            await ctx.send(embed = lib.embedder.make_embed(
                type = "error",
                title = "Shiny Checklist",
                content = "You can't add 0 of something.",
            ))
            return
        elif count < 0:
            await ctx.send(embed = lib.embedder.make_embed(
                type = "error",
                title = "Shiny Checklist",
                content = "There is no such thing as negative Pokémon. At least... not yet.",
            ))
            return

        # Grab the list of Pokemon for the given name or dex number
        pokemon_data = self.get_pokemon_data(pokemon)

        # If no results are returned, tell the user that that Pokemon doesn't
        # exist.
        # If there is more than one returned value... right now it's a WIP and needs to be dealt with
        # If there is only one response returned than work for it
        if not pokemon_data:
            await ctx.send(embed = lib.embedder.make_embed(
                type = "warning",
                title = "Shiny Checklist",
                content = f"Pokémon `{pokemon}` doesn't exist",
            ))
            return
        elif len(pokemon_data) > 1: # WIP Not implemented right now
            await ctx.send(embed = lib.embedder.make_embed(
                type = "warning",
                title = "Shiny Checklist",
                content = "Pokémon with multiple forms, costumes, or variants aren't supported right now.",
            ))
            return
        else:
            db = mysql()
            query = """
                INSERT INTO user_shinies (user_id, dex, type, isotope, count)
                VALUES (%s, %s, %s, %s, %s)
                ON DUPLICATE KEY UPDATE count = count + VALUES(count);
            """
            db.execute(query, [
                ctx.message.author.id,
                pokemon_data[0]['dex'],
                pokemon_data[0]['type'],
                pokemon_data[0]['isotope'],
                count,
            ])
            db.close()

            # Tell the user that they added the Pokemon successfully to their list
            await ctx.send(embed = lib.embedder.make_embed(
                type = "success",
                title = "Shiny Checklist",
                content = f"Added {'a' if count == 1 else count} shiny {pokemon_data[0]['name']} to your list",
                thumbnail = self.generate_image_link(pokemon_data[0], shiny = True)
            ))

    @shiny_group.command(
        name = "remove",
        aliases = ["delete", "r", "d"],
        brief = "Removes a shiny Pokémon from your list",
        description = "Cherubi Bot - Shiny Checklist System (Remove)",
        usage = "<name or dex #> [number]",
        help = "You can give either the name or the dex number of the Pokémon to remove from your list.\n\nYou also can give an amount, if you don't it'll remove a single one."
    )
    async def remove_subcommand(self, ctx, pokemon, count = 1):
        # Just a couple of sanity checks, since I know someone will test this at some point
        if count == 0:
            await ctx.send(embed = lib.embedder.make_embed(
                type = "error",
                title = "Shiny Checklist",
                content = "You can't remove 0 of something.",
            ))
            return
        elif count < 0:
            count = count * -1

        pokemon_data = self.get_pokemon_data(pokemon)
        if not pokemon_data:
            await ctx.send(embed = lib.embedder.make_embed(
                type = "warning",
                title = "Shiny Checklist",
                content = f"Pokémon `{pokemon}` doesn't exist",
            ))
            return
        elif len(pokemon_data) > 1: # WIP Not implemented right now
            await ctx.send(embed = lib.embedder.make_embed(
                type = "warning",
                title = "Shiny Checklist",
                content = "Pokémon with multiple forms, costumes, or variants aren't supported right now.",
            ))
            return
        else:
            db = mysql()
            query = """
                SELECT count
                FROM user_shinies
                WHERE
                    user_id = %s
                    AND dex = %s
                    AND type = %s
                    AND isotope = %s;
            """
            db.execute(query, [
                ctx.message.author.id,
                pokemon_data[0]['dex'],
                pokemon_data[0]['type'],
                pokemon_data[0]['isotope'],
            ])

            # Check if the user has any of the Pokemon in their list. If they
            # don't, tell them, close the DB and then return.
            #
            # Also check if they have the amount they want to remove, if not,
            # set it to what they have
            result = db.fetchone()
            if (not result) or (result['count'] == 0):
                db.close()
                await ctx.send(embed = lib.embedder.make_embed(
                    type = "warning",
                    title = "Shiny Checklist",
                    content = f"You don't have any shiny {pokemon_data[0]['name']} in your list to remove",
                    thumbnail = self.generate_image_link(pokemon_data[0], shiny = True)
                ))
                return
            elif result['count'] < count:
                count = result['count']

            # If they do however, update the count
            query = """
                UPDATE user_shinies
                SET count = count - %s
                WHERE
                    user_id = %s
                    AND dex = %s
                    AND type = %s
                    AND isotope = %s;
            """
            db.execute(query, [
                count,
                ctx.message.author.id,
                pokemon_data[0]['dex'],
                pokemon_data[0]['type'],
                pokemon_data[0]['isotope'],
            ])
            db.close()

            await ctx.send(embed = lib.embedder.make_embed(
                type = "success",
                title = "Shiny Checklist",
                content = f"Removed {count} shiny {pokemon_data[0]['name']} from your list",
                thumbnail = self.generate_image_link(pokemon_data[0], shiny = True)
            ))

    @shiny_group.command(
        name="list",
        aliases=["l"],
        brief="Lists the shiny Pokémon that you have",
        description="Cherubi Bot - Shiny Checklist System (List)",
        help="This lists off all of the shiny Pokémon in your collection."
    )
    @commands.cooldown(1, 10, commands.BucketType.user)
    async def list_subcommand(self, ctx, target: Optional[discord.Member]):
        # If no target is given, use the user who wrote the command
        target = target or ctx.author

        db = mysql()
        query = """
            SELECT
                name.english AS name,
                user_shiny.dex AS dex,
                user_shiny.type AS type,
                user_shiny.isotope AS isotope,
                user_shiny.count AS count
            FROM user_shinies user_shiny
            LEFT JOIN pokemon_names name on name.dex = user_shiny.dex
            WHERE
                user_shiny.user_id = %s
                AND user_shiny.count > 0
            ORDER BY name.english;
        """
        results = db.query(query, [target.id])
        db.close()

        # If the user doesn't have any shiny Pokemon in their list, tell them
        # that
        if not results:
            await ctx.send(embed=lib.embedder.make_embed(
                type="error",
                title="Shiny Checklist",
                content=f"Unfortunately {target.display_name} doesn't have \
any Pokémon in your shiny list...",
            ))

        else:
            total_count = 0
            output = ""
            for result in results:
                output += f"{result['name']}: {result['count']}\n"
                total_count += result['count']

            fields = [
                ("Total:", total_count, False),
            ]

            await ctx.send(embed=lib.embedder.make_embed(
                type="info",
                title=f"{target.display_name}'s Shiny List:",
                content=output,
                fields=fields,
            ))

    def get_pokemon_data(self, input):
        db = mysql()
        query = """
            SELECT
                pkmn.dex AS 'dex',
                name.english AS 'name',
                pkmn.type AS 'type',
                pkmn.isotope AS 'isotope',
                pkmn.filename AS 'filename',
                pkmn.shiny AS 'shiny'
            FROM pokemon pkmn
            LEFT JOIN pokemon_names name on name.dex = pkmn.dex
            WHERE (
                pkmn.dex = %s OR
                name.chinese = %s OR
                name.english = %s OR
                name.french = %s OR
                name.german = %s OR
                name.italian = %s OR
                name.japanese = %s OR
                name.korean = %s OR
                name.portuguese = %s OR
                name.spanish = %s OR
                name.thai = %s
            );
        """
        db.execute(query, [input, input, input, input, input, input, input, input, input, input, input])
        results = db.fetchall()
        db.close()

        return results

    def generate_image_link(self, result, shiny = False):
        # Base url for the repo, plus an image cacher link, if we are using it
        base_url = "https://raw.githubusercontent.com/PokeMiners/pogo_assets/master/Images/Pokemon/"

        url = ""
        url += base_url
        url += "pokemon_icon_"

        # Checks if a unique file name exists for the Pokemon
        if result['filename'] == None: # If no specific file name is given
            # Give it some leading zeroes
            dex = str(result['dex']).zfill(3)

            # base_url + pokemon_icon_{dex}{type}{isotope or ''}_shiny.png
            url += f"{dex}_{result['type']}"

            # If there's an isotope value, add it
            if result['isotope']:
                url += f"_{result['isotope']}"

        else:
            # base_url + pokemon_icon_{fn}_shiny.png
            url += result['filename']

        # If it's shiny, add in that little bit
        if shiny:
            url += "_shiny"

        # Finally, add in the file extension
        url += ".png"

        return url

def setup(client):
    client.add_cog(Checklist(client))


import logging
import os
import math
import logging
import sys
import glob
from natsort import natsorted

import torchvision
import numpy as np
import cv2
from PIL import Image
import torch
from torch.utils.data import Dataset
from torchvision import transforms
from torch.nn import functional as F

from PIL import ImageFile
ImageFile.LOAD_TRUNCATED_IMAGES = True

FORMAT = '[%(levelname)s: %(filename)s: %(lineno)4d]: %(message)s'
logging.basicConfig(level=logging.INFO, format=FORMAT, stream=sys.stdout)
logger = logging.getLogger(__name__)

def cv2pil(image):
    ''' OpenCV -> PIL '''
    new_image = image.copy()
    if new_image.ndim == 2:
        pass
    elif new_image.shape[2] == 3:
        new_image = cv2.cvtColor(new_image, cv2.COLOR_BGR2RGB)
    elif new_image.shape[2] == 4:
        new_image = cv2.cvtColor(new_image, cv2.COLOR_BGRA2RGBA)
    new_image = Image.fromarray(new_image)
    return new_image

def datadir_to_videodata(data_dir):
    return data_dir.replace('annotations', 'videos') + '/video.mov'

def save_obs_img(movie, img_num, img_list, img_dir):
    Fs = int(movie.get(cv2.CAP_PROP_FRAME_COUNT))
    frame_number = 0
    for i in range(Fs - 1):
        flag, frame = movie.read()
        if i in img_num:
            image = cv2pil(frame)
            image.save('{}/frame_num_'.format(img_dir)+'{0:04d}'.format(frame_number)+'.png', quality=95)
            frame_number += 1

def save_obs_crop_img(movie, img_num, img_list, img_dir, seq_list, img_size, obs_len, norm):
    Fs = int(movie.get(cv2.CAP_PROP_FRAME_COUNT))
    frame_number = 0
    per_sum = 0
    for i in range(Fs - 1):
        flag, frame = movie.read()
        if i in img_num:
            image = cv2pil(frame)
            w = image.size[0]/norm
            h = image.size[1]/norm
            k = 0
            for person in seq_list[frame_number]:
                pos = person[:, obs_len-1]
                x_1 = pos[0].item()*w-(img_size//2)
                y_1 = pos[1].item()*h-(img_size//2)
                x_2 = pos[0].item()*w+(img_size//2)
                y_2 = pos[1].item()*h+(img_size//2)
                img_crop = image.crop((x_1, y_1, x_2, y_2))
                img_crop.save('{}/frame_num_'.format(img_dir)+'{0:06d}'.format(per_sum+k)+'.png', quality=95)
                k += 1
            frame_number += 1
            per_sum += k

def seq_collate(data):
    (obs_seq_list, pred_seq_list, obs_seq_rel_list, pred_seq_rel_list, non_linear_ped_list, loss_mask_list, sa_data, pa_data, img) = zip(*data)

    _len = [len(seq) for seq in obs_seq_list]
    cum_start_idx = [0] + np.cumsum(_len).tolist()
    seq_start_end = [[start, end] for start, end in zip(cum_start_idx, cum_start_idx[1:])]

    # Data format: batch, input_size, seq_len
    # LSTM input format: seq_len, batch, input_size
    obs_traj = torch.cat(obs_seq_list, dim=0).permute(2, 0, 1)
    pred_traj = torch.cat(pred_seq_list, dim=0).permute(2, 0, 1)
    obs_traj_rel = torch.cat(obs_seq_rel_list, dim=0).permute(2, 0, 1)
    pred_traj_rel = torch.cat(pred_seq_rel_list, dim=0).permute(2, 0, 1)
    non_linear_ped = torch.cat(non_linear_ped_list)
    sa_data = torch.cat(sa_data, dim=0)
    pa_data = torch.cat(pa_data, dim=0)
    loss_mask = torch.cat(loss_mask_list, dim=0)
    seq_start_end = torch.LongTensor(seq_start_end)
    # img = torch.cat(img, dim=0)
    out = [
        obs_traj, pred_traj, obs_traj_rel, pred_traj_rel, non_linear_ped,
        loss_mask, seq_start_end, sa_data, pa_data, img
    ]
    return tuple(out)

def test_seq_collate(data):
    (obs_seq_list, pred_seq_list, obs_seq_rel_list, pred_seq_rel_list, non_linear_ped_list, loss_mask_list, sa_data, pa_data, img, test_img) = zip(*data)

    _len = [len(seq) for seq in obs_seq_list]
    cum_start_idx = [0] + np.cumsum(_len).tolist()
    seq_start_end = [[start, end] for start, end in zip(cum_start_idx, cum_start_idx[1:])]

    # Data format: batch, input_size, seq_len
    # LSTM input format: seq_len, batch, input_size
    obs_traj = torch.cat(obs_seq_list, dim=0).permute(2, 0, 1)
    pred_traj = torch.cat(pred_seq_list, dim=0).permute(2, 0, 1)
    obs_traj_rel = torch.cat(obs_seq_rel_list, dim=0).permute(2, 0, 1)
    pred_traj_rel = torch.cat(pred_seq_rel_list, dim=0).permute(2, 0, 1)
    non_linear_ped = torch.cat(non_linear_ped_list)
    loss_mask = torch.cat(loss_mask_list, dim=0)
    seq_start_end = torch.LongTensor(seq_start_end)
    sa_data = torch.cat(sa_data, dim=0)
    pa_data = torch.cat(pa_data, dim=0)
    # img = torch.cat(img, dim=0)
    out = [
        obs_traj, pred_traj, obs_traj_rel, pred_traj_rel, non_linear_ped,
        loss_mask, seq_start_end, sa_data, pa_data, img, test_img
    ]
    return tuple(out)

def ph_seq_collate(data):
    (obs_seq_list, pred_seq_list, obs_seq_rel_list, pred_seq_rel_list, non_linear_ped_list, loss_mask_list, sa_data, pa_data, img) = zip(*data)

    _len = [len(seq) for seq in obs_seq_list]
    cum_start_idx = [0] + np.cumsum(_len).tolist()
    seq_start_end = [[start, end] for start, end in zip(cum_start_idx, cum_start_idx[1:])]

    # Data format: batch, input_size, seq_len
    # LSTM input format: seq_len, batch, input_size
    obs_traj = torch.cat(obs_seq_list, dim=0).permute(2, 0, 1)
    pred_traj = torch.cat(pred_seq_list, dim=0).permute(2, 0, 1)
    obs_traj_rel = torch.cat(obs_seq_rel_list, dim=0).permute(2, 0, 1)
    pred_traj_rel = torch.cat(pred_seq_rel_list, dim=0).permute(2, 0, 1)
    non_linear_ped = torch.cat(non_linear_ped_list)
    sa_data = torch.cat(sa_data, dim=0)
    pa_data = torch.cat(pa_data, dim=0)
    loss_mask = torch.cat(loss_mask_list, dim=0)
    seq_start_end = torch.LongTensor(seq_start_end)
    img = torch.cat(img, dim=0)
    out = [
        obs_traj, pred_traj, obs_traj_rel, pred_traj_rel, non_linear_ped,
        loss_mask, seq_start_end, sa_data, pa_data, img
    ]
    return tuple(out)

def ph_test_seq_collate(data):
    (obs_seq_list, pred_seq_list, obs_seq_rel_list, pred_seq_rel_list, non_linear_ped_list, loss_mask_list, sa_data, pa_data, img, test_img) = zip(*data)

    _len = [len(seq) for seq in obs_seq_list]
    cum_start_idx = [0] + np.cumsum(_len).tolist()
    seq_start_end = [[start, end] for start, end in zip(cum_start_idx, cum_start_idx[1:])]

    # Data format: batch, input_size, seq_len
    # LSTM input format: seq_len, batch, input_size
    obs_traj = torch.cat(obs_seq_list, dim=0).permute(2, 0, 1)
    pred_traj = torch.cat(pred_seq_list, dim=0).permute(2, 0, 1)
    obs_traj_rel = torch.cat(obs_seq_rel_list, dim=0).permute(2, 0, 1)
    pred_traj_rel = torch.cat(pred_seq_rel_list, dim=0).permute(2, 0, 1)
    non_linear_ped = torch.cat(non_linear_ped_list)
    loss_mask = torch.cat(loss_mask_list, dim=0)
    seq_start_end = torch.LongTensor(seq_start_end)
    sa_data = torch.cat(sa_data, dim=0)
    pa_data = torch.cat(pa_data, dim=0)
    img = torch.cat(img, dim=0)
    out = [
        obs_traj, pred_traj, obs_traj_rel, pred_traj_rel, non_linear_ped,
        loss_mask, seq_start_end, sa_data, pa_data, img, test_img
    ]
    return tuple(out)

def sophie_seq_collate(data):
    (obs_seq_list, pred_seq_list, obs_seq_rel_list, pred_seq_rel_list, non_linear_ped_list, loss_mask_list, sa_data, pa_data, img) = zip(*data)

    _len = [len(seq) for seq in obs_seq_list]
    cum_start_idx = [0] + np.cumsum(_len).tolist()
    seq_start_end = [[start, end] for start, end in zip(cum_start_idx, cum_start_idx[1:])]

    # Data format: batch, input_size, seq_len
    # LSTM input format: seq_len, batch, input_size
    obs_traj = torch.cat(obs_seq_list, dim=0).permute(3, 0, 1, 2)
    pred_traj = torch.cat(pred_seq_list, dim=0).permute(2, 0, 1)
    obs_traj_rel = torch.cat(obs_seq_rel_list, dim=0).permute(3, 0, 1, 2)
    pred_traj_rel = torch.cat(pred_seq_rel_list, dim=0).permute(2, 0, 1)
    non_linear_ped = torch.cat(non_linear_ped_list)
    sa_data = torch.cat(sa_data, dim=0)
    pa_data = torch.cat(pa_data, dim=0)
    loss_mask = torch.cat(loss_mask_list, dim=0)
    seq_start_end = torch.LongTensor(seq_start_end)
    # img = torch.cat(img, dim=0)
    out = [
        obs_traj, pred_traj, obs_traj_rel, pred_traj_rel, non_linear_ped,
        loss_mask, seq_start_end, sa_data, pa_data, img
    ]
    return tuple(out)

def sophie_test_seq_collate(data):
    (obs_seq_list, pred_seq_list, obs_seq_rel_list, pred_seq_rel_list, non_linear_ped_list, loss_mask_list, sa_data, pa_data, img, test_img) = zip(*data)

    _len = [len(seq) for seq in obs_seq_list]
    cum_start_idx = [0] + np.cumsum(_len).tolist()
    seq_start_end = [[start, end] for start, end in zip(cum_start_idx, cum_start_idx[1:])]

    # Data format: batch, input_size, seq_len
    # LSTM input format: seq_len, batch, input_size
    obs_traj = torch.cat(obs_seq_list, dim=0).permute(3, 0, 1, 2)
    pred_traj = torch.cat(pred_seq_list, dim=0).permute(2, 0, 1)
    obs_traj_rel = torch.cat(obs_seq_rel_list, dim=0).permute(3, 0, 1, 2)
    pred_traj_rel = torch.cat(pred_seq_rel_list, dim=0).permute(2, 0, 1)
    non_linear_ped = torch.cat(non_linear_ped_list)
    loss_mask = torch.cat(loss_mask_list, dim=0)
    seq_start_end = torch.LongTensor(seq_start_end)
    sa_data = torch.cat(sa_data, dim=0)
    pa_data = torch.cat(pa_data, dim=0)
    # img = torch.cat(img, dim=0)
    out = [
        obs_traj, pred_traj, obs_traj_rel, pred_traj_rel, non_linear_ped,
        loss_mask, seq_start_end, sa_data, pa_data, img, test_img
    ]
    return tuple(out)

def read_file(_path, img_path, norm=0, skip=1, delim='\t'):
    data = []
    img = Image.open(img_path)
    cord2imcord = np.array([1.,1.])

    if norm != 0:
        # print(max(img.size[0], img.size[1]))
        w = img.size[0]/norm
        h = img.size[1]/norm
        # w = k
        # h = k
    if delim == 'tab':
        delim = '\t'
    elif delim == 'space':
        delim = ' '
    with open(_path, 'r') as f:
        for line in f:
            line = line.strip().split(delim)
            if str(line[6]) == '0':
                line = [float(i) for i in line[:-4]]
                if line[5] % skip == 0:
                    if norm != 0:
                        line = [line[5]//skip, line[0], (line[3]+line[1])/2/w, (line[4]+line[2])/2/h]
                        if (line[2])/2/w > norm or (line[3])/2/h > norm:
                            print('error')
                    else:
                        line = [line[5]//skip, line[0], (line[3]+line[1])/2, (line[4]+line[2])/2]
                    data.append(line)
    return np.asarray(data), cord2imcord

def poly_fit(traj, traj_len, threshold):
    """
    Input:
    - traj: Numpy array of shape (2, traj_len)
    - traj_len: Len of trajectory
    - threshold: Minimum error to be considered for non linear traj
    Output:
    - int: 1 -> Non Linear 0-> Linear
    """
    t = np.linspace(0, traj_len - 1, traj_len)
    # print(t, traj[0, -traj_len:])
    res_x = np.polyfit(t, traj[0, -traj_len:], 2, full=True)[1]
    res_y = np.polyfit(t, traj[1, -traj_len:], 2, full=True)[1]
    if res_x + res_y >= threshold:
        return 1.0
    else:
        return 0.0

class TrajectoryDataset(Dataset):
    """Dataloder for the Trajectory datasets"""
    def __init__(
        self, args, data_dir, obs_len=8, pred_len=12, center_crop=False, crop_img_size=512, skip=1, skip_frame=2, threshold=0.002,
        min_ped=1, max_ped=1000, delim='\t', norm=1, large_image=False, remake_data=False, test=False, check_so_at=False
    ):
        """
        Args:
        - data_dir: List of dataset paths. [path1, path2, ...]
        <frame_id> <ped_id> <x> <y>

        **stanford dataset : <ped_id> <x_min> <y_min> <x_max> <y_max> <frame_id> <lost> <occluded> <generate> <label>
            lost: If 1, the annotation is outside of the view screen.
            occluded: If 1, the annotation is occluded.
            generated: If 1, the annotation was automatically interpolated.
            label: The label for this annotation, enclosed in quotation marks.

        - obs_len: Number of time-steps in input trajectories
        - pred_len: Number of time-steps in output trajectories
        - center_crop
        - skip: Number of frames to skip while making the dataset
        - threshold: Minimum error to be considered for non linear traj
        - min_ped: Minimum number of pedestrians that should be in a seqeunce
        - max_ped
        - delim: Delimiter in the dataset files
        - norm
        - remake_data
        - test
        """
        super(TrajectoryDataset, self).__init__()

        self.obs_len = obs_len
        self.pred_len = pred_len
        self.skip = skip
        self.seq_len = self.obs_len + self.pred_len
        self.delim = delim
        self.norm = norm
        self.center_crop = center_crop
        self.test = test
        self.check_so_at = check_so_at
        self.ph_type = args.physical_attention_type
        self.large_image = large_image

        if self.large_image:
            self.large_image_size = 640
            self.transform = transforms.Compose(
                [transforms.Resize((self.large_image_size, self.large_image_size)),
                transforms.ToTensor(),
                ])
        else:
            size = 224
            self.transform = transforms.Compose(
                [transforms.Resize((size,size)),
                transforms.ToTensor(),
                # transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
                ])

        num_peds_in_seq = []
        seq_list = []
        s_seq_list = []
        seq_list_rel = []
        s_seq_list_rel = []
        loss_mask_list = []
        non_linear_ped = []
        img_list = []
        test_img_list = []
        self.f_skip = 12
        first = True
        sa = False
        pa = False

        for num in range(len(data_dir)):
            self.remake_data = remake_data
            dn = 0

            self.data_dir = data_dir[num]
            self.img_path = self.data_dir + '/reference.jpg'
            # img = Image.open(self.img_path)
            self.video = cv2.VideoCapture(datadir_to_videodata(self.data_dir))

            img_num = []
            pednum_list = []
            all_files = os.listdir(self.data_dir)
            all_files = [os.path.join(self.data_dir, 'annotations.txt')]

            for path in all_files:
                data, cord2imcord = read_file(path, self.img_path, self.norm, self.f_skip, self.delim)
                frames = np.unique(data[:, 0]).tolist()
                frame_data = []
                for frame in frames:
                    frame_data.append(data[frame == data[:, 0], :])
                num_sequences = int(math.ceil((len(frames)  - self.seq_len + 1) / self.skip))
                # print(len(frame_data))
                # print(num_sequences)

                for idx in range(0, num_sequences * self.skip + 1, self.skip):
                    if len(frame_data[idx:idx + self.seq_len]) < 1:
                        continue
                    # curr_seq_data = np.concatenate(f_data[::self.f_skip], axis=0)
                    curr_seq_data = np.concatenate(frame_data[idx:idx + self.seq_len], axis=0)
                    peds_in_curr_seq = np.unique(curr_seq_data[:, 1])
                    curr_seq_rel = np.zeros((len(peds_in_curr_seq), 2, self.seq_len))
                    curr_seq = np.zeros((len(peds_in_curr_seq), 2, self.seq_len))
                    curr_loss_mask = np.zeros((len(peds_in_curr_seq),self.seq_len))

                    num_peds_considered = 0
                    _non_linear_ped = []

                    for _, ped_id in enumerate(peds_in_curr_seq):
                        curr_ped_seq = curr_seq_data[curr_seq_data[:, 1] == ped_id, :]
                        pad_front = frames.index(curr_ped_seq[0, 0]) - idx
                        pad_end = frames.index(curr_ped_seq[-1, 0]) - idx + 1
                        # print(pad_end,pad_front)
                        if pad_end - pad_front != self.seq_len:
                            continue
                        if curr_ped_seq.shape[0] != self.seq_len:
                            continue
                        curr_ped_seq = np.transpose(curr_ped_seq[:, 2:])
                        # print(curr_ped_seq)
                        # Make coordinates relative
                        rel_curr_ped_seq = np.zeros(curr_ped_seq.shape)
                        rel_curr_ped_seq[:, 1:] = curr_ped_seq[:, 1:] - curr_ped_seq[:, :-1]
                        _idx = num_peds_considered
                        # print(pad_front,pad_end)
                        # print(curr_ped_seq.shape)
                        curr_seq[_idx, :, pad_front:pad_end] = curr_ped_seq
                        curr_seq_rel[_idx, :, pad_front:pad_end] = rel_curr_ped_seq
                        # Linear vs Non-Linear Trajectory
                        _non_linear_ped.append(poly_fit(curr_ped_seq, pred_len, threshold))
                        curr_loss_mask[_idx, pad_front:pad_end] = 1
                        num_peds_considered += 1
                        # print(num_peds_considered)

                        if num_peds_considered == max_ped:
                            break

                    if num_peds_considered > min_ped:
                        dn += 1
                        non_linear_ped += _non_linear_ped
                        num_peds_in_seq.append(num_peds_considered)
                        loss_mask_list.append(curr_loss_mask[:num_peds_considered])
                        seq_list.append(curr_seq[:num_peds_considered])
                        # print(seq_list)
                        seq_list_rel.append(curr_seq_rel[:num_peds_considered])
                        # img_list.append(self.transform(img))
                        img_num.append((idx+self.obs_len)*self.f_skip)
                        pednum_list.append(num_peds_considered)
                        if args.social_attention_type == 'sophie' or args.physical_attention_type == 'sophie':
                            num_peds_in_seq.append(num_peds_considered)
                            curr_seq_exp = np.zeros((num_peds_considered, args.n_max, 2, self.seq_len))
                            curr_seq_rel_exp = np.zeros((num_peds_considered, args.n_max, 2, self.seq_len))
                            for i in range(num_peds_considered):
                                curr_seq_exp[i, 0, :, :] = curr_seq[i]
                                curr_seq_exp[i, 1:i+1, :, :] = curr_seq[0:i]
                                curr_seq_exp[i, i+1:num_peds_considered, :, :] = curr_seq[i+1:num_peds_considered]

                                dists = (curr_seq_exp[i, :] - curr_seq_exp[i, 0]) ** 2
                                dists = np.sum(np.sum(dists, axis=2), axis=1)
                                idxs = np.argsort(dists)
                                curr_seq_exp[i, :] = curr_seq_exp[i, :][idxs]
                                curr_seq_rel_exp[i, 0, :, :] = curr_seq_rel[i]
                                curr_seq_rel_exp[i, 1:i+1, :, :] = curr_seq_rel[0:i]
                                curr_seq_rel_exp[i, i+1:num_peds_considered, :, :] = curr_seq_rel[i+1:num_peds_considered]
                                curr_seq_rel_exp[i, :] = curr_seq_rel_exp[i, :][idxs]
                            s_seq_list.append(curr_seq_exp[:num_peds_considered])
                            s_seq_list_rel.append(curr_seq_rel_exp[:num_peds_considered])

            # if self.center_crop:
            #     img_dir = "{}/crop_True_cropsize_{}_skip_{}_obs_{}_pre_{}_minped_{}".format(self.data_dir, crop_img_size, skip, obs_len, pred_len, min_ped)
            # else:
            img_dir = "{}/crop_False_skip_{}_obs_{}_pre_{}_minped_{}_fskip_{}".format(self.data_dir, self.skip, obs_len, pred_len, min_ped, self.f_skip)

            if not os.path.exists(img_dir):
                os.mkdir(img_dir)
                self.remake_data = True

            if self.remake_data:
                logger.info('Remake image data: {}_{}'.format(self.data_dir.split('/')[-2], self.data_dir.split('/')[-1]))
                # if not self.center_crop:
                save_obs_img(self.video, img_num, img_list, img_dir)
                # else:
                #     save_obs_crop_img(self.video, img_num, img_list, img_dir, seq_list, crop_img_size, self.obs_len, norm)

            if self.test or args.physical_attention_dim != 0:
                files = glob.glob(os.path.join(img_dir, '*.png'))

                if self.ph_type == 'prior3' or self.ph_type == 'prior4' or self.ph_type == 'prior5' or self.ph_type == 'prior6' or self.ph_type == 'self_attention':
                    n = 0
                    # path = self.data_dir + '/image_2.csv'
                    # img_save = []
                    for path in natsorted(files):
                        ped_num = pednum_list[n]
                        img = Image.open(path)
                        img2 = self.transform(img)
                        if self.test:
                            test_img_list.append(img)
                        for _ in range(ped_num):
                            img_list.append(img2)
                        img.load()

                        n += 1
                else:
                    for path in natsorted(files):
                        img = Image.open(path)
                        img_list.append(self.transform(img))
                        if self.test:
                            test_img_list.append(img)
                        img.load()

            # if self.test and self.center_crop:
            #     test_img_dir = "{}/crop_False_skip_{}_obs_{}_pre_{}_minped_{}".format(self.data_dir, skip, obs_len, pred_len, min_ped)
            #     test_files = glob.glob(os.path.join(test_img_dir, '*.png'))
            #     for path in natsorted(test_files):
            #         img = Image.open(path)
            #         test_img_list.append(img)
            #         img.load()

            if args.social_attention_dim != 0 and args.social_attention_type =='prior':
                sa = True
                # f_path = self.data_dir + '/pseudo-social_attention_.csv'
                if args.so_ver == 1:
                    f_path = self.data_dir + '/pseudo-social_attention.csv'
                elif args.so_ver == 2:
                    f_path = self.data_dir + '/pseudo-social_attention_gauss.csv'
                else:
                    print("pseudo social attention version ERROR")

                if first:
                    self.sa_data = np.loadtxt(f_path)
                else:
                    self.sa_data = np.concatenate([self.sa_data , np.loadtxt(f_path)])

            if args.physical_attention_dim != 0 and (args.physical_attention_type !='one_head_attention' and args.physical_attention_type !='simple' and args.physical_attention_type !='one_head_attention2' and args.physical_attention_type !='sat' and args.physical_attention_type !="NICG_sat" and args.physical_attention_type != 'sophie') and args.ph_prior_type != 'one_head_attention' and args.ph_prior_type != 'not_prior':
                pa = True
                if args.ph_ver == 1:
                    g_path = self.data_dir + '/pseudo-physical_attention_mask.csv'
                elif args.ph_ver == 2:
                    g_path = self.data_dir + '/pseudo-physical_attention_fan_mask.csv'
                elif args.ph_ver == 3:
                    g_path = self.data_dir + '/pseudo-physical_attention_leaky_mask.csv'
                elif args.ph_ver == 4:
                    g_path = self.data_dir + '/pseudo-physical_attention_gauss1.csv'
                elif args.ph_ver == 5:
                    g_path = self.data_dir + '/pseudo-physical_attention_gauss2.csv'
                elif args.ph_ver == 6:
                    g_path = self.data_dir + '/pseudo-physical_attention_gauss3.csv'
                elif args.ph_ver == 7:
                    g_path = self.data_dir + '/pseudo-physical_attention_gauss4.csv'
                else:
                    print("pseudo physical attention version ERROR")

                if first:
                    self.pa_data = np.loadtxt(g_path)
                else:
                    self.pa_data = np.concatenate([self.pa_data , np.loadtxt(g_path)])

            logger.info('{}/{} is done. {}'.format(self.data_dir.split('/')[-2], self.data_dir.split('/')[-1], dn))
            first = False

        # print(self.a_data.shape)
        if args.social_attention_type == 'sophie' or args.physical_attention_type == 'sophie':
            s_seq_list = np.concatenate(s_seq_list, axis=0)
            s_seq_list_rel = np.concatenate(s_seq_list_rel, axis=0)

        self.num_seq = len(seq_list)
        seq_list = np.concatenate(seq_list, axis=0)
        seq_list_rel = np.concatenate(seq_list_rel, axis=0)
        loss_mask_list = np.concatenate(loss_mask_list, axis=0)
        non_linear_ped = np.asarray(non_linear_ped)

        # Convert numpy -> Torch Tensor
        if args.social_attention_type == 'sophie' or args.physical_attention_type == 'sophie':
            self.obs_traj = torch.from_numpy(s_seq_list[:, :, :, :self.obs_len]).type(torch.float)
            self.obs_traj_rel = torch.from_numpy(s_seq_list_rel[:, :, :, :self.obs_len]).type(torch.float)
        else:
            self.obs_traj = torch.from_numpy(seq_list[:, :, :self.obs_len]).type(torch.float)
            self.obs_traj_rel = torch.from_numpy(seq_list_rel[:, :, :self.obs_len]).type(torch.float)
        self.pred_traj = torch.from_numpy(seq_list[:, :, self.obs_len:]).type(torch.float)
        self.pred_traj_rel = torch.from_numpy(seq_list_rel[:, :, self.obs_len:]).type(torch.float)
        self.loss_mask = torch.from_numpy(loss_mask_list).type(torch.float)

        self.non_linear_ped = torch.from_numpy(non_linear_ped).type(torch.float)
        cum_start_idx = [0] + np.cumsum(num_peds_in_seq).tolist()
        self.seq_start_end = [
            (start, end) for start, end in zip(cum_start_idx, cum_start_idx[1:])
        ]
        if args.physical_attention_dim != 0:
            self.img_data = torch.stack(img_list, dim=0)
        else:
            self.img_data = self.pred_traj_rel

        if self.test:
            self.test_img_data = test_img_list

        if sa:
            self.sa_data = torch.from_numpy(self.sa_data).type(torch.float)
        else:
            self.sa_data = self.pred_traj_rel

        if pa:
            self.pa_data = torch.from_numpy(self.pa_data).type(torch.float)
        else:
            self.pa_data = self.pred_traj_rel

        print(self.img_data.shape, self.pa_data.shape, self.sa_data.shape)

    def __len__(self):
        return self.num_seq

    def __getitem__(self, index):
        start, end = self.seq_start_end[index]
        # print(self.obs_traj.shape)
        if self.check_so_at:
            pd_num = end - start
            b = True
            # obs_traj: batch, xy, t
            obs_traj = self.obs_traj[start:end, :]
            last_pos = self.obs_traj[start:end, :, -1]
            _obs_traj = torch.zeros(3,2,8)
            _pred_traj = torch.zeros(3,2,12)
            _obs_traj_rel = torch.zeros(3,2,8)
            _pred_traj_rel = torch.zeros(3,2,12)
            _non_linear_ped = torch.zeros(3)
            _loss_mask = torch.zeros(3, 20)

            list_pd = np.arange(pd_num)
            a = 0
            r = 0.005

            while b:
                q = list_pd[a]
                d = (last_pos[:,0] - last_pos[q,0].item())**2 + (last_pos[:,1] - last_pos[q,1].item())**2
                # print(d.shape)
                dl = d.clone()
                dl = torch.sort(dl)
                # print(dl[1])
                if dl[0][1] <= r:
                    b = False
                else:
                    a += 1
                    if a == pd_num:
                        a = 0
                        r += 0.005
            min = dl[1][1]
            max = dl[1][-1]
            _obs_traj[0] = self.obs_traj[start+a, :].unsqueeze(0)
            _obs_traj[1] = self.obs_traj[start+min, :].unsqueeze(0)
            _obs_traj[2] = self.obs_traj[start+max, :].unsqueeze(0)
            _pred_traj[0] = self.pred_traj[start+a, :].unsqueeze(0)
            _pred_traj[1] = self.pred_traj[start+min, :].unsqueeze(0)
            _pred_traj[2] = self.pred_traj[start+max, :].unsqueeze(0)
            _obs_traj_rel[0] = self.obs_traj_rel[start+a, :].unsqueeze(0)
            _obs_traj_rel[1] = self.obs_traj_rel[start+min, :].unsqueeze(0)
            _obs_traj_rel[2] = self.obs_traj_rel[start+max, :].unsqueeze(0)
            _pred_traj_rel[0] = self.pred_traj_rel[start+a, :].unsqueeze(0)
            _pred_traj_rel[1] = self.pred_traj_rel[start+min, :].unsqueeze(0)
            _pred_traj_rel[2] = self.pred_traj_rel[start+max, :].unsqueeze(0)
            _non_linear_ped[0] = self.non_linear_ped[start+a]
            _non_linear_ped[1] = self.non_linear_ped[start+min]
            _non_linear_ped[2] = self.non_linear_ped[start+max]
            _loss_mask[0] = self.loss_mask[start+a, :].unsqueeze(0)
            _loss_mask[1] = self.loss_mask[start+min, :].unsqueeze(0)
            _loss_mask[2] = self.loss_mask[start+max, :].unsqueeze(0)

            if self.test:
                out = [
                        _obs_traj, _pred_traj, _obs_traj_rel, _pred_traj_rel, _non_linear_ped, _loss_mask,
                        self.sa_data[3*index:3*index+3], self.pa_data[start:start+3, :],
                        self.img_data[index], self.test_img_data[index]
                    ]
            else:
                out = [
                        _obs_traj, _pred_traj, _obs_traj_rel, _pred_traj_rel, _non_linear_ped, _loss_mask,
                        self.sa_data[3*index:3*index+3], self.pa_data[start:start+3, :], self.img_data[index]
                    ]

        elif self.ph_type == 'prior3' or self.ph_type == 'prior4' or self.ph_type == 'prior5' or self.ph_type == 'prior6' or self.ph_type == 'self_attention':
            if self.test:
                out = [
                    self.obs_traj[start:end, :], self.pred_traj[start:end, :],
                    self.obs_traj_rel[start:end, :], self.pred_traj_rel[start:end, :],
                    self.non_linear_ped[start:end], self.loss_mask[start:end, :],
                    self.sa_data[start:end, :], self.pa_data[start:end],
                    self.img_data[start:end], self.test_img_data[index]
                ]

            else:

                out = [
                    self.obs_traj[start:end, :], self.pred_traj[start:end, :],
                    self.obs_traj_rel[start:end, :], self.pred_traj_rel[start:end, :],
                    self.non_linear_ped[start:end], self.loss_mask[start:end, :],
                    self.sa_data[start:end, :], self.pa_data[start:end], self.img_data[start:end]
                ]

        else:
            if self.test:
                out = [
                    self.obs_traj[start:end, :], self.pred_traj[start:end, :],
                    self.obs_traj_rel[start:end, :], self.pred_traj_rel[start:end, :],
                    self.non_linear_ped[start:end], self.loss_mask[start:end, :],
                    self.sa_data[start:end, :], self.pa_data[start:end],
                    self.img_data[index], self.test_img_data[index]
                ]

            else:

                out = [
                    self.obs_traj[start:end, :], self.pred_traj[start:end, :],
                    self.obs_traj_rel[start:end, :], self.pred_traj_rel[start:end, :],
                    self.non_linear_ped[start:end], self.loss_mask[start:end, :],
                    self.sa_data[start:end, :], self.pa_data[start:end], self.img_data[index]
                ]
        return out


from docutils import nodes
from docutils.parsers.rst import Directive, directives

import requests

class Contributor:
    """Main class for the contributors atributes."""

    def __init__(self, login, url, contributions=0):
        """Initialize atributes."""
        self.contributions = contributions
        self.login = login
        self.url = url
        self.contributions = contributions

    def build(self):
        """Build contributor details."""
        node_contributor = nodes.paragraph()
        node_contributor += nodes.reference(text=self.login, refuri=self.url)
        node_contributor += nodes.Text(' - ' + str(self.contributions) + ' ' +
                                       ('contributions' if self.contributions != 1 else 'contribution'))
        return node_contributor

class ContributorsRepository:
    """Main class for repo atributes."""

    def __init__(self, contributors, reverse=True, limit=10, exclude=None):
        """Initialize repo details."""
        self.contributors = sorted([c for c in contributors if c.login not in exclude],
                                   key=lambda c: c.contributions,
                                   reverse=reverse)[:limit]

    def build(self):
        """Build contributors repo details."""
        node_list = nodes.bullet_list()
        for contributor in self.contributors:
            node_contributor = nodes.list_item()
            node_contributor += contributor.build()
            node_list += node_contributor
        return node_list

class ContributorsDirective(Directive):
    """Main class for the contributors directive."""

    has_content = True
    required_arguments = 1
    optional_arguments = 0
    final_argument_whitespace = True
    option_spec = {
        'limit': directives.positive_int,
        'order': directives.unchanged,
        'exclude': directives.unchanged,
    }

    def run(self):
        """Run the plugin."""
        limit = self.options.get('limit', 10)
        order = self.options.get('order', 'DESC') == 'DESC'
        exclude = self.options.get('exclude', '').split(",")

        r = requests.get('https://api.github.com/repos/' + self.arguments[0] + '/contributors?per_page=100')
        if type(r.json()) == dict:
            raise ValueError('The repository ' + self.arguments[0] + ' does not exist.')
        contributors = list(map(lambda c: Contributor(c.get('login'),
                                                      c.get('html_url'),
                                                      c.get('contributions')), r.json()))
        return [ContributorsRepository(contributors, reverse=order, limit=limit, exclude=exclude).build()]

def setup(app):
    """Configure the plugin."""
    app.add_directive('ghcontributors', ContributorsDirective)

    return {'version': '0.1'}


# /home/jay/anaconda3/bin/python

import numpy as np
from sklearn.metrics import log_loss
from scipy.special import expit

# sig = lambda x: 1.0/(1.0+np.exp(-x))
sig = expit
sig_d = lambda x: sig(x) * (1 - sig(x))
sig_to_d = lambda x: x * (1 - x)
# log_loss = lambda y,yhat: np.sum(-(y*np.log(yhat) + (1 - y)*np.log(1-yhat)))

def print_shape(f):
    print(list(map(lambda x: x.shape, f)))

def get_shape(f):
    return str(list(map(lambda x: x.shape, f)))

class mlp:

    def __init__ (self, layer_sizes):
        '''
        Parameters
        ----------
        layer_sizes : list of integers
            sizes of the layres in the network. first number is the input layer
            last number is the output layer
        '''
        self.weights = [np.random.randn(x, y)
                        for x, y in zip(layer_sizes[:-1], layer_sizes[1:])]

        self.biases = [np.random.randn(x) for x in layer_sizes[1:]]

    def __str__(self):
        out = 'weights:' + get_shape(self.weights)
        out += ', biases:' + get_shape(self.biases)
        return out

    def feed_forward(self, X, dropout=False, dropout_percent=0.5):
        '''
        Parameters
        ----------
        X : numpy.array shape(records, features)

        Returns
        -------
        a : numpy.array
            output of neural network

        '''
        assert X.shape[1] == self.weights[0].shape[0], 'input X is wrong shape'
        a = X
        self.z = []
        self.a = [a]
        l = len(self.weights)
        i = 0
        for w, b in zip(self.weights, self.biases):
            z = np.dot(a, w) + b
            a = sig(z)
            if dropout and i > 0 and i != l-1:
                drop_mat = np.random.binomial(
                                [np.ones(z.shape)], 1-dropout_percent)[0] * (
                                1.0/(1-dropout_percent)
                            )
                z *= drop_mat
                a *= drop_mat

            self.z.append(z)
            self.a.append(a)
            i += 1

        return a

    def back_propigation(self, X, Y, dropout=False, dropout_percent=0.5):
        self.feed_forward(X, dropout, dropout_percent)
        ddw = []
        ddb = []
        delta = -(Y - self.a[-1]) * sig_to_d(self.a[-1])

        for i in reversed(range(len(self.weights))):
            ddw.append(np.dot(self.a[i].T, delta))
            ddb.append(delta)
            if i != 0:
                delta = np.dot(delta, self.weights[i].T) * sig_to_d(self.a[i])

        return list(reversed(ddw)), [b.sum(axis = 0) for b in reversed(ddb)]

    def gradient_descent(self, X, Y, itterations, batch_size, a = 1e-2,
                         dropout=False, dropout_percent=0.5):
        '''
        Parameters
        ----------

        Returns
        -------
        '''
        assert batch_size <= X.shape[0]
        assert X.shape[0] == Y.shape[0], 'X and Y different lengths'

        n_batches = X.shape[0] // batch_size
        batches = np.array_split(range(X.shape[0]), n_batches)

        for i in range(itterations):
            for b in batches:
                ddw, ddb = self.back_propigation(X[b], Y[b],
                                                 dropout, dropout_percent)
                for j in range(len(self.weights)):
                    self.weights[j] -= ddw[j] * a
                    self.biases[j] -= ddb[j] * a

    def predict(self, X):
        '''
            feed_forward without recording activations or using dropout
            Parameters
            ----------
                X : numpy.array shape(records, features)

            Returns
            -------
                a : numpy.array
                    output of neural network
        '''
        assert X.shape[1] == self.weights[0].shape[0], 'input X is wrong shape'
        a = X
        for w, b in zip(self.weights, self.biases):
            z = np.dot(a, w) + b
            a = sig(z)
        return a

    def gradient_checking(self, X, Y):
        '''
        utility function to check back_propigation
        '''
        bp = self.back_propigation(X, Y)[0]
        yhat = self.feed_forward(X)
        # print_shape(bp)
        weights = self.weights[:]
        epsilon = 1e-4
        # epsilon = 1
        grad_approx = [np.zeros(w.shape) for w in self.weights]

        for i in range(len(self.weights)):

            for j, x in np.ndenumerate(self.weights[i]):
                zeros_mat = np.zeros(self.weights[i].shape)
                # theta_plus
                zeros_mat[j] += epsilon
                self.weights[i] = self.weights[i] + zeros_mat
                p1 = np.array_equal(weights[i], self.weights[i])
                theta_plus = self.feed_forward(X)
                self.weights[i] = weights[i][:]
                # theta_minus
                zeros_mat = np.zeros(self.weights[i].shape)
                zeros_mat[j] = epsilon
                self.weights[i] = self.weights[i] - zeros_mat
                p2 = np.array_equal(weights[i], self.weights[i])
                theta_minus = self.feed_forward(X)
                # reset weights
                self.weights[i] = weights[i][:]
                p3 = np.array_equal(weights[i], self.weights[i])

                if any([p1, p2,  not p3]):
                    print(p1, p2, p3)
                    print(i, j)

                # print((log_loss(Y, theta_plus),
                #       log_loss(Y, theta_minus)), (2 * epsilon))
                grad_approx[i][j] = (log_loss(Y, theta_plus) -
                      log_loss(Y, theta_minus)) / (2 * epsilon)

        # for i,j in zip(bp, grad_approx):
        #     print(i - j)

        return grad_approx

def main():
    from sklearn.datasets import load_digits
    from sklearn.preprocessing import OneHotEncoder
    from sklearn.cross_validation import train_test_split
    from sklearn.metrics import classification_report
    import pandas as pd

    fp = '/home/jay/projects/mlp/'

    df = pd.read_csv(fp + 'train.csv')

    X, Y = df.drop('label', 1).values, df['label'].values
    onehot = OneHotEncoder(sparse = False)
    Y = onehot.fit_transform(np.atleast_2d(Y).T)

    train_x, test_x, train_y, test_y = train_test_split( X, Y,
        test_size = 0.3)

    test_y = np.argmax(test_y, 1)
    train_y2 = np.argmax(train_y, 1)

    nn = mlp([X.shape[1], 100, 10])
    # nn = mlp([X.shape[1], 100, 10])
    yhat = np.argmax(nn.feed_forward(train_x), 1)
    # nn.gradient_descent(train_x, train_y, 10, 100, a = 1e-1)
    # nn.gradient_descent(train_x, train_y, 9000, 100, a = 1e-2)
    # nn.gradient_descent(train_x, train_y, 50, 10, a=1e-1,
    #                     dropout=False, dropout_percent=0.5)

    # nn.gradient_descent(train_x, train_y, 50, 10, a=1e-2,
    #                     dropout=False, dropout_percent=0.5)

    nn.gradient_descent(train_x, train_y, 500, 10, a=1e-3,
                        dropout=True, dropout_percent=0.5)

    nn.gradient_descent(train_x, train_y, 500, 10, a=1e-4,
                        dropout=True, dropout_percent=0.5)

    yhat = np.argmax(nn.predict(test_x), 1)
    print(nn)
    print(classification_report(test_y, yhat))

if __name__ == '__main__':
    main()

    # nn = mlp([2, 2, 1])
    #
    # X = np.array([[0,0],
    #               [0,1],
    #               [1,0],
    #               [1,1]])
    #
    # Y = np.array([[0, 1, 1, 0]]).T
    #
    # nn.gradient_descent(X, Y, 1000, 4, dropout=True)
    # for x in [(i[0],j[0]) for i,j in zip(nn.predict(X), Y)]:
    #     print(x)


#!../python35/python.exe
print ("Content-type: text/html\n")
import cgi
import cgitb; cgitb.enable()
form = cgi.FieldStorage()
import pymysql
conn = pymysql.connect(host='localhost', port=3306, user='root', passwd='',db='soccer',autocommit=True)
cur = conn.cursor()
nombre = form.getfirst("nombre");
posicion = form.getfirst("posicion");
equipo = form.getfirst("equipo");
numero = form.getfirst("numero");
import os

fileitem = form['img']
if fileitem.filename:
   fn = os.path.basename(fileitem.filename)
   open("images/player/" +fn, 'wb').write(fileitem.file.read())
   imagen = fn
else:
   imagen = "None"

print(nombre,posicion,equipo,numero)
sql = "INSERT INTO `jugadores`(`nombre_jugador`, `id_equipo`, `posicion`, `numero`,foto) VALUES ('"+str(nombre)+"','"+str(equipo)+"','"+str(posicion)+"','"+str(numero)+"','"+str(imagen)+"')"
cur.execute(sql)
print('''<body><script type="text/javascript">window.location="admin_jugadores.py";</script></body>''')


import nextcord
from nextcord.ext import commands
from nextcord import Interaction, SlashOption
from config import settings

class ExampleSlash(commands.Cog):
    def __init__(self, bot):
        self.bot = bot

    @nextcord.slash_command(guild_ids=[settings.GUILD_ID], description="Чистка текущего канала от сообщений")
    async def clear(self, interaction: Interaction, amount: int=SlashOption(description="Количество сообщений для удаления", required=True)):
        await interaction.response.send_message(f"Ваш запрос выполняется. Пожалуйста, подождите...", ephemeral=True)
        messages = await interaction.channel.purge(limit=amount)
        await interaction.edit_original_message(content=f"Удалено **{len(messages)}** сообщений в этом канале")

def setup(bot):
    bot.add_cog(ExampleSlash(bot))
    print("Module ExampleSlash successfully loaded")


from django.urls import path, include
from django.conf import settings
from django.conf.urls.static import static

from .views import index, PostDetail, news_all, ContactView, TeamDetail, TeamAll, club, CategoryNews, Galery, subscription, \
    locations, SubscriptionView

urlpatterns = [
    path('', index, name='index'),
    path('subscription/buy/', SubscriptionView.as_view(), name='subscription'),
    path('locations/', locations, name='locations'),
    path('subscription/', subscription, name='sub'),
    path('galery/', Galery.as_view(), name='galery'),
    path('category/<str:slug>/', CategoryNews.as_view(), name='category'),
    path('club/', club, name='club'),
    path('team/', TeamAll.as_view(), name='team_all'),
    path('team_detail/<int:pk>/', TeamDetail.as_view(), name='team'),
    path('post/<str:slug>/', PostDetail.as_view(), name='detail'),
    path('news_all/', news_all, name='news_all'),
    path('contact/', ContactView.as_view(), name='contact'),
]


import cv2
import os

src_dir = 'H:\\document\\ocr\\src\\shixin2'
dst_dir = 'H:\\document\\ocr\\dst-test'

files = os.listdir(src_dir)

for name in files:
    im = cv2.imread(os.path.join(src_dir, name))
    im_gray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
    blurred = cv2.GaussianBlur(im_gray, (5, 5), 0)
    ret, thresh = cv2.threshold(blurred, 220, 255, cv2.THRESH_BINARY)
    cv2.imwrite(os.path.join(dst_dir, name), thresh)
print('done')


import os
import json
import logging

from slacker.logger import Logger
from slacker.session import Session

# Default REPL prompt template string. Override with "repl_prompt" field in
# .slacker config file.
#
# Supported identifers:
# ${ro} - display DEFAULT_READ_ONLY_STR when Slacker is running in read-only mode
# ${w} - current workspace Slacker is working against
DEFAULT_REPL_PROMPT = "${ro}${w}> "

# Default string used when Slacker is running in read-only mode. Override with
# read_only_str in .slacker config file.
DEFAULT_READ_ONLY_STR = "(read-only) "

class Config:
  __instance = None

  def __init__(self):
    """Get instance of Config via Config.get()."""
    if not Config.__instance:
      self.__logger = Logger(self.__class__.__name__).get()

      # Set default values.
      self.reset()

      # Load from file if it exists.
      try:
        self.load()
      except Exception as ex:
        self.__logger.debug("Config does not exist: {}\n{}".format(self.file_path(), ex))

      # Assign singleton instance.
      Config.__instance = self

  @staticmethod
  def get():
    if not Config.__instance:
      Config()
    return Config.__instance

  def repl_prompt(self):
    return self.__repl_prompt

  def set_repl_prompt(self, repl_prompt):
    self.__repl_prompt = repl_prompt

  def active_workspace(self):
    """Active workspace name, if defined."""
    return self.__active_workspace

  def set_active_workspace(self, name):
    if name not in self.__workspaces:
      raise ValueError("Cannot set unknown workspace active: '{}'".format(name))
    self.__active_workspace = name

  def add_workspace(self, name, token):
    if name in self.__workspaces:
      raise ValueError("Cannot add workspace '{}' because it already exists!".format(name))
    self.__workspaces[name] = token

  def remove_workspace(self, name):
    if name not in self.__workspaces:
      raise ValueError("Cannot remove unknown workspace: '{}'".format(name))
    if self.active_workspace() == name:
      raise ValueError("Cannot remove active workspace: '{}'".format(name))
    del(self.__workspaces[name])

  def workspaces(self):
    return list(self.__workspaces.keys())

  def workspace_token(self, name):
    if name not in self.__workspaces:
      raise ValueError("Cannot get token for unknown workspace: '{}'".format(name))
    return self.__workspaces[name]

  def active_workspace_token(self):
    if not self.active_workspace():
      raise ValueError("No workspace is active!")
    return self.workspace_token(self.active_workspace())

  def set_log_level(self, level):
    if level not in Logger.levels():
      raise ValueError("Invalid log level: {}".format(level))
    self.__log_level = level
    Session.get().set_log_level(level)
    Logger.set_level(level)

  def log_level(self):
    return self.__log_level

  def read_only(self):
    return self.__read_only

  def set_read_only(self, enable):
    self.__read_only = enable

  def read_only_str(self):
    return self.__read_only_str

  def set_read_only_str(self, str):
    self.__read_only_str = str

  def file_path(self):
    return os.path.expanduser("~/.slacker")

  def safe_dict(self):
    """Returns a safe dictionary of current values excluding any tokens."""
    return {"repl_prompt": self.repl_prompt(),
            "workspaces": self.workspaces(),
            "active_workspace": self.active_workspace(),
            "log_level": self.log_level(),
            "read_only": self.read_only(),
            "read_only_str": self.read_only_str()}

  def save(self):
    data = {"repl_prompt": self.repl_prompt(),
            "workspaces": self.__workspaces,
            "active_workspace": self.active_workspace(),
            "log_level": self.log_level(),
            "read_only": self.read_only(),
            "read_only_str": self.read_only_str()}
    with open(self.file_path(), "w") as fp:
      json.dump(data, fp, indent=2)
      self.__logger.debug("Saved config to: {}".format(self.file_path()))

  def load(self):
    with open(self.file_path(), "r") as fp:
      data = json.load(fp)
      if "repl_prompt" in data:
        self.set_repl_prompt(data["repl_prompt"])
      if "workspaces" in data:
        self.__workspaces = data["workspaces"]
      if "active_workspace" in data:
        self.__active_workspace = data["active_workspace"]
      if "log_level" in data:
        self.set_log_level(data["log_level"])
      if "read_only" in data:
        self.set_read_only(data["read_only"])
      if "read_only_str" in data:
        self.set_read_only_str(data["read_only_str"])
      self.__logger.debug("Loaded config from: {}".format(self.file_path()))

  def reset(self):
    """Resets all values to default."""
    self.set_repl_prompt(DEFAULT_REPL_PROMPT)
    self.__workspaces = {}  # Workspace name -> API token
    self.__active_workspace = None
    self.__log_level = logging.INFO
    self.__read_only = False
    self.__read_only_str = DEFAULT_READ_ONLY_STR


from dataclasses import dataclass, field
from dataclasses_json import dataclass_json
from typing import Tuple, List, Set, Optional, Dict, Iterable
from math import sqrt, asin, sin, cos, pi
from pathlib import Path

Coordinate = Tuple[float, float]

def distance(p1: Coordinate, p2: Coordinate):
    return sqrt((p1[0] - p2[0])**2 + (p1[1] - p2[1])**2)

R_EARTH = 3959.0
def gc_distance(geo1: Coordinate, geo2: Coordinate, R: float = R_EARTH):
    lat1, lon1 = geo1[0] * pi/180, geo1[1] * pi/180
    lat2, lon2 = geo2[0] * pi/180, geo2[1] * pi/180
    rhs = (sin((lat2-lat1)/2)**2) + cos(lat1)*cos(lat2)*(sin((lon2-lon1)/2)**2)
    return 2*R*asin(sqrt(rhs))

@dataclass_json
@dataclass
class MapPoint:
    index: int               # Numbered ordering of points
    dxf_coords: Coordinate   # Coordinates taken from raw DXF file
    city_names: List[str] = field(default_factory=list)
        # Use a list for pairs like San Francisco-Oakland
        # Will only be populated for named destination cities
    geonames_lookup: Optional[str] = field(default=None)
        # Name to lookup in geonames dump data
    place_name: Optional[str] = field(default=None)
        # Single place name for displays
    state: Optional[str] = field(default=None)
        # May be unknown before lat,lon are calculated
    geo_coords: Optional[Coordinate] = field(default=None)
        # Latitude/longitude (need to be calculated)
    connections: Dict[str, Set[int]] = field(default_factory=dict)
        # Indexes of connected MapPoints, grouped by railroad name
    final_svg_coords: Optional[Coordinate] = field(default=None)
        # Final coordinates in the .svg
    region: Optional[str] = None

    @property
    def display_name(self) -> str:
        return f'{(self.place_name or "").replace("_","")}, {self.state}'

    def _connect_to(self, other: 'MapPoint', rr: str):
        if rr not in self.connections:
            self.connections[rr] = set()
        self.connections[rr].add(other.index)

    def connect_to(self, other: 'MapPoint', rr: str):
        self._connect_to(other, rr)
        other._connect_to(self, rr)

    @property
    def pts_connected_to(self) -> List[int]:
        return list(sorted(set(
            pt_j for pts in self.connections.values() for pt_j in pts)))

@dataclass_json
@dataclass
class Railroad:
    shortName: str
    longName: str
    cost: int
    triangles: List[Tuple[int, int, int]] | None = None

@dataclass_json
@dataclass
class Map:
    points: List[MapPoint] = field(default_factory=list)
    map_transform_A: List[List[float]] = field(default_factory=list)
    map_transform_b: List[float] = field(default_factory=list)
    railroads: Dict[str, Railroad] = field(default_factory=dict)

    def map_transform(self, c: Coordinate) -> Coordinate:
        x, y = c
        A, b = self.map_transform_A, self.map_transform_b
        return A[0][0] * x + A[0][1] * y + b[0], \
               A[1][0] * x + A[1][1] * y + b[1]

    def lookup_city(self, city: str) -> Tuple[str, int]:
        def canon(s: str) -> str:
            return s.upper().replace('.','').replace(' ','')
        for pt in self.points:
            for c in pt.city_names:
                if canon(c) == canon(city):
                    return c, pt.index
        raise StopIteration

    def gc_distance(self, pt_i: int, pt_j: int, R: float = R_EARTH) -> float:
        geo1 = self.points[pt_i].geo_coords
        geo2 = self.points[pt_j].geo_coords
        assert geo1, "Must have geo coords"
        assert geo2, "Must have geo coords"
        return gc_distance(geo1, geo2, R)

def read_map(json_path: Path | None = None) -> Map:
    if not json_path:
        json_path = (Path(__file__) / '../../../../../data/map.json').resolve()
    with json_path.open('r') as json_file:
        return Map.from_json(json_file.read()) # type: ignore

Waypoint = Tuple[str, int]         # Railroad name, dot
RailSegment = Tuple[str, int, int] # Railroad name + 2 dots (in order)

def make_rail_seg(rr: str, pt_i: int, pt_j: int) -> RailSegment:
    assert pt_i != pt_j, "Rail segment must connect two different points"
    return (rr, pt_i, pt_j) if pt_i < pt_j else (rr, pt_j, pt_i)

def rail_segs_from_wps(start_pt: int, wp: List[Waypoint]) -> List[RailSegment]:
    curr_pt = start_pt
    rail_segs: List[RailSegment] = []
    for rr, next_pt in wp:
        assert curr_pt != next_pt, f"Invalid history {wp} from {start_pt} -> duplicate {curr_pt}"
        rail_segs.append(make_rail_seg(rr, curr_pt, next_pt)); curr_pt = next_pt
    return rail_segs

def get_valid_waypoints(m: Map, pt_i: int,
        exclude_rs: List[RailSegment] = [],
        exclude_pts: Iterable[int] = []) -> List[Waypoint]:
    wps: List[Waypoint] = []
    for rr, conn_pts in m.points[pt_i].connections.items():
        for pt_j in conn_pts:
            rs = make_rail_seg(rr, pt_i, pt_j)
            if pt_j not in exclude_pts and rs not in exclude_rs:
                wps.append((rr, pt_j))
    return wps


from django.shortcuts import render
from django.shortcuts import Http404
from django.shortcuts import HttpResponse
from crawler.tools import crawler,indexJson,search
from django.views.decorators.csrf import csrf_protect
from json import dumps,loads
# Create your views here.

@csrf_protect
def indexPage(request):
    context = {}
    return render(request, 'indexePage.html', context)

@csrf_protect
def searchPage(request):
    context = {}
    return render(request, 'searchPage.html', context)

@csrf_protect
def searchIt(request):
    if request.method != 'POST':
        raise Http404
    searchTerm = request.POST['searchTerm']
    result = search(searchTerm)
    result = dumps(result)
    print("result",result)
    return HttpResponse(result, content_type='application/json')

@csrf_protect
def indexIt(request):
    if request.method != 'POST':
        raise Http404
    url = request.POST['url']
    count = request.POST['count']
    count = int(count)
    print("crawling on pages count " , count, " starting form page: ",url )
    result = crawler(url , count)
    indexJson()
    result = [{'url':item['url'], 'rank':item['rank']} for item in result['content']]
    result = dumps(result)
    return HttpResponse(result, content_type='application/json')


""" Contains a pipeline for generating a data set for use with the LightTag
platform

see: https://www.lighttag.io/
"""
import json

from ucla_topic_analysis import get_file_list
from ucla_topic_analysis.data import get_training_file_path
from ucla_topic_analysis.data.coroutines import create_file
from ucla_topic_analysis.data.coroutines import insert
from ucla_topic_analysis.data.coroutines import print_progress
from ucla_topic_analysis.data.pipeline import Pipeline
from ucla_topic_analysis.data.coroutines.read import ReadFilePipeline

class LightTagDataSetPipeline(Pipeline):
    """Pipeline for generating a dataset for the LightTag platform.

     NOTE: This pipeline is a data sink. It does not return any new data.
    """

    # The split schema for the corpus files
    SCHEMA = {
        "training": 0.8,
        "validation": 0.1,
        "testing": 0.1
    }

    @staticmethod
    def get_input_stream(schema=None):
        """This function is used to get an input stream for the
        LightTagDataSetPipeline

        Args:
            schema(:obj:`dict`): The schema for the file pipeline

        Returns:
            An iterable containing the dataset.
        """
        # Build the input stream pipeline
        files = ReadFilePipeline.get_input_stream()
        return ReadFilePipeline(
            input_stream=files,
            schema=schema
        ).output_stream()

    @classmethod
    async def generate_dataset(cls):
        """This function is used to create a dataset for the LightTag platform
        """
        #build the pipeline
        data_stream = cls.get_input_stream(cls.SCHEMA)
        pipeline = cls()

        # create the dataset
        count = 1
        total = len(get_file_list())
        async for data in data_stream:
            await pipeline.run(data)
            print_progress(count, total)
            count += 1
        print("")

    async def coroutine(self, data):
        """This function dictionaries to a json file for using in LightTag
        data sets.

        Args:
            data (:obj:`dict`): A dictionary containing data that needs to be
                tagged
        """
        file_path = get_training_file_path("LightTag-dataset.json")
        is_new_file = create_file(file_path, "[\n]")
        data_string = json.dumps(data)
        prefix = "\n" if is_new_file else ",\n"
        insertion_string = "{0}{1}".format(prefix, data_string)
        with open(file_path, "r+") as json_file:
            json_file.seek(0, 2)
            position = json_file.tell() - 2
            insert(insertion_string, json_file, position)


from dash import Dash, dcc, Output, Input  # pip install dash
import dash_bootstrap_components as dbc    # pip install dash-bootstrap-components
import plotly.express as px
import pandas as pd                        # pip install pandas
import geopandas as gpd

df = pd.read_csv("rainfall.csv")
print(df.head())

gdf = gpd.read_file('Indian_States.txt')
gdf["geometry"] = gdf.to_crs(gdf.estimate_utm_crs()).simplify(1000).to_crs(gdf.crs)
india_states = gdf.rename(columns={"NAME_1": "ST_NM"}).__geo_interface__

# Build your components
app = Dash(__name__, external_stylesheets=[dbc.themes.LUX])
mytitle = dcc.Markdown(children='')
mygraph = dcc.Graph(figure={})
dropdown = dcc.Dropdown(options=df.columns.values[1:],
                        value='Rainfall(mm)',  # initial value displayed when page first loads
                        clearable=False)

# Customize your own Layout
app.layout = dbc.Container([
    dbc.Row([
        dbc.Col([mytitle], width=6)
    ], justify='center'),
    dbc.Row([
        dbc.Col([mygraph], width=12)
    ]),
    dbc.Row([
        dbc.Col([dropdown], width=6)
    ], justify='center'),

], fluid=True)

# Callback allows components to interact
@app.callback(
    Output(mygraph, 'figure'),
    Output(mytitle, 'children'),
    Input(dropdown, 'value')
)
def update_graph(column_name):  # function arguments come from the component property of the Input

    print(column_name)
    print(type(column_name))
    fig = px.choropleth(
        pd.json_normalize(india_states["features"])["properties.ST_NM"],
        locations="properties.ST_NM",
        geojson=india_states,
        featureidkey="properties.ST_NM",
        color_discrete_sequence=["lightgrey"],
        )

    fig.add_traces(
        px.choropleth(
            df,
            locations="State",
            geojson=india_states,
            featureidkey="properties.ST_NM",
            locationmode="geojson-id",
            color=column_name,
            scope="asia",
        ).data
    )
    fig.update_geos(fitbounds="locations", visible=False)

    return fig, '# '+column_name  # returned objects are assigned to the component property of the Output

# Run app
if __name__=='__main__':
    app.run_server(debug=True, port=8054)


import pykka
from site_storage.sсhema import RegularCheck, WatchStatus
from site_storage.messages import UpdateSiteRequest
from site_storage.messages import SiteDeleteResponse, SiteResponse, SubscribeOnSiteUpdates
import os
import urllib.request
from urllib.parse import urldefrag
import time
import threading
from datetime import datetime
import logging

logger = logging.getLogger(__name__)

class Handler(threading.Thread):
    def __init__(self, downloader):
        super().__init__(daemon=True)
        self.running = True
        self.downloader = downloader

    def run(self):
        last_update = datetime.now()
        while self.running:
            current_time = datetime.now()
            difference = current_time - last_update
            if difference.seconds >= 60:
                self.downloader._recalculate_queue()
                last_update = datetime.now()

            if len(self.downloader.queue) > 0:
                site = self.downloader.queue.pop()
                self.downloader._download_site(site[1])

            time.sleep(5)

    def stop(self):
        self.running = False
        print('Site downloader stopped')

class SiteDownloaderActor(pykka.ThreadingActor):
    point_dict = {
        RegularCheck.TWICE_HOUR: 1000,
        RegularCheck.ONCE_HOUR: 500,
        RegularCheck.FOUR_TIMES_DAY: 250,
        RegularCheck.TWICE_DAY: 125,
        RegularCheck.ONCE_DAY: 100
    }

    regular_check_duration = {
        RegularCheck.TWICE_HOUR: 30,
        RegularCheck.ONCE_HOUR: 60,
        RegularCheck.FOUR_TIMES_DAY: 360,
        RegularCheck.TWICE_DAY: 720,
        RegularCheck.ONCE_DAY: 1440
    }

    def __init__(self, storage_proxy, analytic_proxy):
        try:
            super().__init__()
            self.storage_proxy = storage_proxy
            self.analytic_proxy = analytic_proxy
            self.sites = dict()
            self.queue = list()
            self._create_queue()
            self.storage_proxy.subscribe_on_site_update(
                    SubscribeOnSiteUpdates(self.actor_ref)
            )
            self.handle_process = Handler(downloader=self)
        except Exception as e:
            logger.error("Error occurred: {0}".format(e))

    def on_start(self):
        self.handle_process.start()

    def on_stop(self):
        self.handle_process.stop()

    def _create_queue(self):
        for site in self.storage_proxy.get_sites().get():
            self.sites[site.id] = site
            points = self._calculate_priority(site)
            if points > 0:
                self.queue.append((points, site))
        self.queue.sort(key=lambda x: x[0])

    def _recalculate_queue(self):
        for site in self.sites.values():
            points = self._calculate_priority(site)
            if points > 0:
                if site.last_watch is not None:
                    self.storage_proxy.update_site(UpdateSiteRequest(id=site.id, status=WatchStatus.NEED_TO_WATCH))

                self.queue.append((points, site))
        self.queue.sort(key=lambda x: x[0])

    def _calculate_priority(self, site):
        points = 0

        points = points + self.point_dict[site.regular_check]
        if site.last_watch is None:
            points = points + 10000
        else:
            difference = datetime.now() - datetime.fromisoformat(site.last_watch)
            fine = (difference.seconds // 60) - self.regular_check_duration[site.regular_check]
            if fine < 0:
                points = -1
            else:
                points = points + fine * 100

        return points

    def on_receive(self, message):
        if isinstance(message, SiteResponse):
            self.on_site_record(message)

        if isinstance(message, SiteDeleteResponse):
            self.on_delete_site(message)

    def on_site_record(self, site):
        self.sites[site.id] = site

    def on_delete_site(self, message):
        del self.sites[message.id]

    def _download_site(self, site):
        try:
            self.storage_proxy.update_site(UpdateSiteRequest(id=site.id, status=WatchStatus.IN_PROGRESS))
            with urllib.request.urlopen(site.url) as f:
                html = f.read().decode('utf-8')
                self.analytic_proxy.analyze_site(site, html)
        except Exception as e:
            logger.error("Error occurred: {0}".format(e))
            self.storage_proxy.update_site(UpdateSiteRequest(id=site.id, status=WatchStatus.ERROR))