import argparse
import os

import torch
from torch.utils.data import DataLoader
from egg import core
import neptune
from neptunecontrib.api.utils import get_filepaths

from compositionality.games import PretrainingmGameReinforce, PretrainingmGameGS, CompositionalGameGS, CompositionalGameReinforce
from compositionality.wrappers import RnnReceiverDeterministic, RnnReceiverGS
from compositionality.callbacks import NeptuneMonitor, EarlyStopperAccuracy, CompositionalityMetricReinforce, CompositionalityMetricGS
from compositionality.agents import Sender, Receiver, VisualSender
from compositionality.data import prepare_datasets

from visual_compositionality.visual_data import prepare_datasets

def get_params():
    parser = argparse.ArgumentParser()
    parser.add_argument('--n_features', type=int, default=5,
                        help='Dimensionality of the "concept" space (default: 10)')
    parser.add_argument('--n_attributes', type=int, default=2,
                        help='Number of attributes (default: 2')
    parser.add_argument('--sender_hidden', type=int, default=200,
                        help='Size of the hidden layer of Sender (default: 200)')
    parser.add_argument('--receiver_hidden', type=int, default=200,
                        help='Size of the hidden layer of Receiver (default: 200)')
    parser.add_argument('--sender_embedding', type=int, default=50,
                        help='Dimensionality of the embedding hidden layer for Sender (default: 10)')
    parser.add_argument('--receiver_embedding', type=int, default=50,
                        help='Dimensionality of the embedding hidden layer for Receiver (default: 10)')
    parser.add_argument('--rnn_cell', type=str, default='rnn')
    parser.add_argument('--sender_lr', type=float, default=0.001,
                        help="Learning rate for Sender's parameters (default: 1e-3)")
    parser.add_argument('--receiver_lr', type=float, default=0.001,
                        help="Learning rate for Receiver's parameters (default: 1e-3)")
    parser.add_argument('--seed', type=int, default=171,
                        help="Random seed")
    parser.add_argument('--pretrain', action='store_true', default=False, help="")
    parser.add_argument('--reinforce', action='store_true', default=False, help="")
    parser.add_argument('--config', type=str, default=None)

    args = core.init(parser)
    print(args)
    return args

if __name__ == "__main__":
    opts = get_params()
    opts.on_slurm = os.environ.get('SLURM_JOB_NAME', False)
    core.util._set_seed(opts.seed)
    full_dataset, train, test = prepare_datasets(opts.n_features, opts.n_attributes)
    train_loader = DataLoader(train, batch_size=opts.batch_size, drop_last=False, shuffle=True)
    test_loader = DataLoader(test, batch_size=opts.batch_size, drop_last=False, shuffle=False)

    if opts.reinforce:
        pretrained_senders = [
            core.RnnSenderReinforce(
                agent=Sender(opts.sender_hidden, opts.n_features, opts.n_attributes),
                vocab_size=opts.vocab_size,
                embed_dim=opts.sender_embedding,
                hidden_size=opts.sender_hidden,
                max_len=1,
                cell=opts.rnn_cell,
                force_eos=False
            )
            for i in range(2)]
        sender_3 = core.RnnSenderReinforce(
            agent=Sender(opts.sender_hidden, opts.n_features, opts.n_attributes),
            vocab_size=opts.vocab_size,
            embed_dim=opts.sender_embedding,
            hidden_size=opts.sender_hidden,
            max_len=opts.max_len,
            force_eos=False,
            cell=opts.rnn_cell)
        receiver = RnnReceiverDeterministic(
            agent=Receiver(opts.receiver_hidden, opts.n_features, opts.n_attributes),
            vocab_size=opts.vocab_size,
            embed_dim=opts.receiver_embedding,
            hidden_size=opts.receiver_hidden,
            cell=opts.rnn_cell)
    else:
        pretrained_senders = [
            core.RnnSenderGS(
                agent=VisualSender(opts.sender_hidden, opts.n_features, opts.n_attributes),
                vocab_size=opts.vocab_size,
                embed_dim=opts.sender_embedding,
                hidden_size=opts.sender_hidden,
                max_len=1,
                temperature=3.,
                trainable_temperature=True,
                cell=opts.rnn_cell,
                force_eos=False
            )
            for i in range(2)]
        sender_3 = core.RnnSenderGS(
                agent=VisualSender(opts.sender_hidden, opts.n_features, opts.n_attributes),
                vocab_size=opts.vocab_size,
                embed_dim=opts.sender_embedding,
                hidden_size=opts.sender_hidden,
                max_len=opts.max_len,
                temperature=3.,
                trainable_temperature=True,
                force_eos=False,
                cell=opts.rnn_cell)
        receiver = RnnReceiverGS(
                agent=Receiver(opts.receiver_hidden, opts.n_features, opts.n_attributes),
                vocab_size=opts.vocab_size,
                embed_dim=opts.receiver_embedding,
                hidden_size=opts.receiver_hidden,
                cell=opts.rnn_cell)

    neptune.init('tomekkorbak/template-transfer')
    with neptune.create_experiment(params=vars(opts), upload_source_files=get_filepaths(), tags=['rrr']) as experiment:
        CompositionalityMetric = CompositionalityMetricReinforce if opts.reinforce else CompositionalityMetricGS

        # Pretraining game
        if opts.pretrain:
            pretraining_game = PretrainingmGameReinforce(pretrained_senders, receiver) if opts.reinforce else PretrainingmGameGS(pretrained_senders, receiver)
            sender_params = [{'params': sender.parameters(), 'lr': opts.sender_lr} for sender in pretrained_senders]
            receiver_params = [{'params': receiver.parameters(), 'lr': opts.receiver_lr}]
            optimizer = torch.optim.Adam(sender_params + receiver_params)
            trainer = core.Trainer(
                game=pretraining_game, optimizer=optimizer, train_data=train_loader,
                validation_data=test_loader,
                callbacks=[
                    CompositionalityMetric(full_dataset, pretrained_senders[0], opts, opts.vocab_size, prefix='1_'),
                    CompositionalityMetric(full_dataset, pretrained_senders[1], opts, opts.vocab_size, prefix='2_'),
                    NeptuneMonitor(prefix='pretrain'),
                    core.ConsoleLogger(print_train_loss=not opts.on_slurm),
                    EarlyStopperAccuracy(threshold=0.85, field_name='accuracy', delay=2),
                ])
            trainer.train(n_epochs=500_000)
            pretraining_game.train(False)

        # Compositional game
        compositional_game = CompositionalGameReinforce(sender_3, receiver) if opts.reinforce else CompositionalGameGS(sender_3, receiver)
        sender_params = [{'params': sender_3.parameters(), 'lr': opts.sender_lr}]
        receiver_params = [{'params': receiver.parameters(), 'lr': opts.receiver_lr * 0.5}]
        optimizer = torch.optim.Adam(sender_params + receiver_params)
        trainer = core.Trainer(game=compositional_game, optimizer=optimizer, train_data=train_loader,
                               validation_data=test_loader,
                               callbacks=[
                                   CompositionalityMetric(full_dataset, sender_3, opts, opts.vocab_size, prefix='comp'),
                                   NeptuneMonitor(prefix='comp'),
                                   core.ConsoleLogger(print_train_loss=not opts.on_slurm),
                               ])
        trainer.train(n_epochs=opts.n_epochs)


import torch
import torch.nn as nn
from mmcv.cnn import ConvModule

from enseg.ops import resize
from enseg.models.builder import DECODE_SEG
from enseg.models.decode_gen.base import BaseGen
from enseg.models.decode_seg.psp_head import PPM

@DECODE_SEG.register_module()
class BackGen(BaseGen):
    """
    输入是seg头的输出特征。一般分割头的处理过程为:feature->high dimension feature(BFHW)->seg logits(BCHW)
    我们取高维特征BFHW, 以及输入原始图像，进行图像翻译
    对于3*256*512图像:
    高维特征512*64*128
    原始图像3*256*512
    3*256*512+++++++++++++++++++++++++++++++|->3*256*512
        |->64*128*256++++++++++++++++++|->64*128*256
            |->128*64*128+512*64*128->128*64*128

    """

    def __init__(
        self, base_dim=64, seg_dim=512, act_cfg=dict(type="LeakyReLU"), **kwargs
    ):
        super(BackGen, self).__init__(
            act_cfg=act_cfg, in_channels=[], in_index=[], channels=1, **kwargs
        )
        self.down_layers = nn.ModuleList()
        self.downdeal_layers = nn.ModuleList()
        self.up_layers = nn.ModuleList()
        self.updeal_layers = nn.ModuleList()
        norm_act_cfg = dict(norm_cfg=self.norm_cfg, act_cfg=self.act_cfg,)
        cur_dim = base_dim
        channels = [
            (3, base_dim),
            (base_dim, base_dim * 2),
        ]
        for depth, channel in enumerate(channels):
            in_ch, out_ch = channel
            downdeal_layer = nn.Sequential(
                ConvModule(in_ch, out_ch // 2, 3, 1, 1, **norm_act_cfg),
                ConvModule(out_ch // 2, out_ch, 3, 1, 1, **norm_act_cfg),
            )
            down_layer = ConvModule(out_ch, out_ch, 4, 2, 1, **norm_act_cfg)
            in_ch = out_ch
            out_ch += out_ch
            self.downdeal_layers.append(downdeal_layer)
            self.down_layers.append(down_layer)
        self.neck = nn.Sequential(
            ConvModule(channels[-1][-1], seg_dim // 2, 3, 1, 1, **norm_act_cfg),
            ConvModule(seg_dim // 2, seg_dim, 3, 1, 1, **norm_act_cfg),
        )
        channels = [
            (seg_dim + seg_dim, base_dim*2),
            (base_dim*2+base_dim*2, base_dim),
            (base_dim + base_dim, base_dim),
        ]
        for depth, channel in enumerate(channels):
            in_ch, out_ch = channel
            updeal_layer = nn.Sequential(
                ConvModule(in_ch, in_ch // 2, 3, 1, 1, **norm_act_cfg),
                ConvModule(in_ch // 2, out_ch, 3, 1, 1, **norm_act_cfg),
            )
            up_layer = ConvModule(
                out_ch, out_ch, 4, 2, 1, **norm_act_cfg, conv_cfg=dict(type="deconv")
            )
            in_ch = out_ch
            out_ch += out_ch
            self.updeal_layers.append(updeal_layer)
            self.up_layers.append(up_layer)
        self.output = nn.Sequential(
            ConvModule(base_dim, base_dim, 7, 1, 3, **norm_act_cfg),
            ConvModule(base_dim, 3, 1, 1, norm_cfg=None, act_cfg=dict(type="Tanh")),
        )

    def forward_model(self, inputs, origin_img):
        """Forward function."""

        seg_feature = inputs
        img = origin_img
        x = img
        downdeal_features = []
        for down_layer, downdeal_layer in zip(self.down_layers, self.downdeal_layers):
            x = downdeal_layer(x)
            downdeal_features.append(x)
            x = down_layer(x)
        x = self.neck(x)
        downdeal_features.append(seg_feature)
        for up_layer, updeal_layer in zip(self.up_layers, self.updeal_layers):
            x = torch.cat([x, downdeal_features.pop()], 1)
            x = updeal_layer(x)
            x = up_layer(x)
        x = self.output(x)
        return x

if __name__ == "__main__":
    BackGen()


import logging
import logging.config
import os
import json

def setup_logging(default_path=os.path.dirname(os.path.realpath(__file__)) + "/../loggerConfiguration.json",
                  default_level=logging.INFO):
    """
    Setup logging configuration

    :param default_path: path to the logger configuration
    :param default_level: defines the default logging level if configuration file is not found (default: logging.INFO)
    """
    path = default_path
    if os.path.exists(path):
        with open(path, 'rt') as f:
            config = json.load(f)
        logging.config.dictConfig(config)
    else:
        logging.basicConfig(level=default_level)

setup_logging()


# -*- encoding: utf-8 -*-
'''
@project :   LeetCode
@File    :   largestNumber.py
@Contact :   9824373@qq.com
@Desc    :
            给定一组非负整数，重新排列它们的顺序使之组成一个最大的整数。

            示例 1:

            输入: [10,2]
            输出: 210
            示例 2:

            输入: [3,30,34,5,9]
            输出: 9534330
            说明: 输出结果可能非常大，所以你需要返回一个字符串而不是整数。

            来源：力扣（LeetCode）
            链接：https://leetcode-cn.com/problems/largest-number
@Modify Time                @Author     @Version    @Desciption
------------                -------     --------    -----------
2020-03-03     zhan        1.0         None
'''
from typing import List
import functools

class Solution:
    def cmp(self,a,b):
        if str(a) + str(b) >str(b) + str(a):
            return -1
        else:
            return 1

    def largestNumber(self, nums: List[int]) -> str:
        if len(nums) == 0: return  '0'
        if len(nums) == 1: return  str(nums[0])

        for i, num in enumerate(nums):
            nums[i] = str(num)

        nums.sort(key = functools.cmp_to_key(self.cmp))
        ans = ''.join(nums) +  '1'
        ans.strip('0')
        if len(ans) == 1:
            ans = '0'
        else:
            ans = ans[0:-1]
        return ans

if __name__ == '__main__':
    a = [3, 30, 34, 5, 9]

    ans = Solution().largestNumber(a)
    print(ans)


from fastapi import FastAPI
from fastapi.responses import HTMLResponse

from app.endpoints import calendar, oauth2

app = FastAPI()
app.include_router(oauth2.router)
app.include_router(calendar.router)

@app.get("/", response_class=HTMLResponse)
def index():
    return """

            <title>SDS App</title>

Hello! Thanks for visiting SDS App!

    """

if __name__ == '__main__':
    from uvicorn import run

    run(app, port=8080)


#!/usr/bin/env python3.7
# coding: utf-8

from __future__ import unicode_literals

from ParsingShow import ParseRunInterfaceDeeperFile
from dictdiffer import diff
import pyparsing as pp
import argparse
from pprint import pprint
import pdb
import ntpath
import re
import sys
import yaml

import os.path, time

def parseSed(Str__):
	resultat=[]
	Sed=pp.Literal('s/').suppress()+pp.Word(pp.alphanums)+pp.Literal('/').suppress()+pp.Word(pp.alphanums)+pp.Literal('/').suppress()+pp.Optional(pp.Literal('g')).suppress()

	try:
		resultat=Sed.parseString(Str__,parseAll=True).asList()
	except pp.ParseException as e:
		print("Erreur parsing l'option replace")
		print("respecter la forme sed pour remplacer s/.../.../")
		sys.exit(1)

	return resultat

def replace_dict(dict__,str__,equipement):
	resultat=str__
	for key__ in dict__[equipement].keys():
		if re.search('[a-zA-Z]'+key__+'$',str__) or re.search('\s'+key__+'$',str__):
			resultat=str__.replace(key__,dict__[equipement][key__])
			break;
	return resultat

def filter_info(list_parser,filter=None,replace="",equipement=""):
	resultat={}

	if replace:
		if filter:
			for entry in list_parser[0].keys():
				if re.search(filter,entry):
					new_entry=replace_dict(replace,entry,equipement)
					resultat[new_entry]=list_parser[0][entry]

		else:
			for entry in list_parser[0].keys():
				new_entry=replace_dict(replace,entry)
				resultat[new_entry]=list_parser[0][entry]
	else:
		if filter:
			for entry in list_parser[0].keys():
				if re.search(filter,entry):
					resultat[entry]=list_parser[0][entry]
		else:
			for entry in list_parser[0].keys():
				resultat[entry]=list_parser[0][entry]

	return resultat

def print_diff(list_diff):
	resultat=[]
	for entry in list_diff:
		resultat.append(entry)

	return resultat

def print_time_file(file__):
	print("last modified "+ntpath.basename(file__)+":  "+time.ctime(os.path.getmtime(file__)))

if __name__ == '__main__':
	"Fonction principale"
	parser = argparse.ArgumentParser()

	parser.add_argument('file1')
	parser.add_argument('file2')
	parser.add_argument("--filter1", action="store",help="filtre sur les interfaces (regex) file 1",required=False)
	parser.add_argument("--filter2", action="store",help="filtre sur les interfaces (regex) file 2",required=False)
	parser.add_argument("-r", "--replace", action="store",help="fichier yaml",required=False)
	parser.add_argument("-e", "--equipement", action="store",default='SC2ASWIPP001',help="fichier yaml",required=False)

	args = parser.parse_args()
	print('\n')
	print_time_file(args.file1)
	print_time_file(args.file2)
	print('\n')

	Remplacement={}
	if args.replace:
		with open(args.replace, 'r' ) as model_yml:
			Remplacement = yaml.load(model_yml,Loader=yaml.SafeLoader)

	result=diff(filter_info(ParseRunInterfaceDeeperFile(args.file1),filter=args.filter1,replace=Remplacement,equipement=args.equipement),filter_info(ParseRunInterfaceDeeperFile(args.file2),filter=args.filter2,equipement=args.equipement))

	pprint(print_diff(result))


from pyspark import Row
from pyspark.sql import SparkSession
from pyspark.sql.functions import expr, regexp_replace

class ProductDimension:
	"""
		Reads dim.csv and
	"""

	def __init__(self, spark_session):
		self.spark = spark_session
		self.sc = self.spark.sparkContext

	def file_to_df(self):
		dim_lines = self.sc.textFile("data/dim.csv")
		header = dim_lines.first()
		dim_lines = dim_lines.filter(lambda r: r != header)
		dim_parts = dim_lines.map(lambda l: l.split(","))

		dim = dim_parts.map(lambda p: Row(
			is_banner=p[1],
			is_tax=p[2],
			is_market_place=p[3],
			material_status=p[4],
			current_price_range=p[5],
			cmc_division=p[6],
			cmc_business_unit=p[7],
			gender=p[8],
			product=p[9]
		))

		return self.spark.createDataFrame(dim)

	@staticmethod
	def gender_transform(df):
		gender = expr("UPPER(CASE WHEN gender IS NULL OR gender = '' THEN 'UNISSEX' ELSE gender END)")
		df = df.withColumn("gender", gender)
		df = df.withColumn("gender", regexp_replace(df['gender'], '(.)*(UNISEX)(.)*', 'UNISSEX'))
		df = df.withColumn("gender", regexp_replace(df['gender'], '(.)*(MASC){1}(.)*(FEM){1}(.)*', 'UNISSEX'))
		df = df.withColumn("gender", regexp_replace(df['gender'], '(.)*(FEM){1}(.)*(MASC){1}(.)*', 'UNISSEX'))
		df = df.withColumn("gender", regexp_replace(df['gender'], '(.)*(BEB)(.)*', 'BEBE'))
		df = df.withColumn(
			"gender", regexp_replace(
				df['gender'], '(.)*(NIÑ|MENIN|BOYS|GIRL|INFANTIL|JUVENIL)(.)*', 'INFANTIL'
			)
		)
		df = df.withColumn("gender", regexp_replace(df['gender'], '(.)*(FEM)(.)*', 'FEMININO'))
		df = df.withColumn("gender", regexp_replace(df['gender'], '(.)*(MASC)(.)*', 'MASCULINO'))

		return df

	def csv_to_tmp_view(self, view_name="dim"):
		df = self.file_to_df()
		df = self.gender_transform(df)

		df.createOrReplaceTempView(view_name)


# -*- coding: utf-8 -*-
"""
Created on Mon Apr  6 13:49:48 2020
@author: Bruna
"""

import sys
import os
import sqlite3
from contextlib import closing

from classes import Item
from classes import Customer

conn = sqlite3.connect("project.sqlite")
conn.row_factory = sqlite3.Row

def connect():
    global conn
    if not conn:
        if sys.platform == "win32":
            DB_FILE = "\Desktop\GBC\WINTER 2020\PYTHON\PROJECT\project.sqlite"
        else:
            HOME = os.environ["HOME"]
            DB_FILE = HOME + "/Documents/murach/python/_db/movies.sqlite"
            conn = sqlite3.connect(DB_FILE)
            conn.row_factory = sqlite3.Row

def close():
    if conn:
        conn.close()

def makeItem(row):
    return Item(row[0], row[1], row[2])

# TO DISPLAY THE ITEMS AVAILABLE IN THE DATABASE
def getItems():
    query = '''SELECT itemID AS ID,
                    name AS Item,
                    price AS Price
                FROM Item'''
    with closing(conn.cursor()) as c:
        c.execute(query)
        results = c.fetchall()

    items = []
    for row in results:
        items.append(makeItem(row))
    return items

# TO GET ONE SPECIFC ITEM IN THE DATABASE
def getItem(itemID):
    query = '''SELECT itemID AS ID, name AS Item, price AS Price
                FROM Item WHERE itemID = ?'''

    with closing(conn.cursor()) as c:
        c.execute(query, (itemID,))
        row = c.fetchone()

    item = makeItem(row)
    return item

# ADD A NEW CUSTOMER IN THE DATABASE
def addCustomer(customer):
    sql = '''INSERT INTO Customer (customerID, name)
            VALUES (?, ?)'''
    with closing(conn.cursor()) as c:
        c.execute(sql, (customer.customerID, customer.name))
        conn.commit()


import os, sys
import subprocess
import argparse

from profiler.profile_op.op_cfg import config2str, str2config
from utils.cutlass_api import OperationArgs, find_cutlass_kernel
from utils.device_info import DEVICE_INFO, short_device_name, query_cc

import tensorflow as tf

parser = argparse.ArgumentParser(prog="Test")
parser.add_argument('--cfg_str', type=str, default=None, help="If set, run one specific config")
parser.add_argument('--ops', type=str, default=None, help="OPs to run")
parser.add_argument('--target_gpu', type=str, default=None, help="target gpu")
parser.add_argument('--cache_path', type=str, default=None, help="Path to cache profile results")
args = parser.parse_args()

op2cutlass_kernel_type = {
    "MatMul": "gemm",
    # "MatMul": "sparsgemm",
    "Conv2D": "conv2d_fprop",
}

DTYPE_NAME_CONVERTER = {
    tf.dtypes.float16: "f16",
    tf.dtypes.float32: "f32"
}

if __name__ == "__main__":
    op_args = OperationArgs()

    op_args.kernel_type = op2cutlass_kernel_type[args.ops]

    capability = query_cc(args.target_gpu)
    op_args.cc_major = capability // 10
    op_args.cc_minor = capability % 10

    config = str2config(args.cfg_str)
    op_args.dtype = DTYPE_NAME_CONVERTER[config["dtype"]]
    op_args.m = config["N"]
    op_args.k = config["C_in"]
    op_args.n = config["C_out"]
    kernel = find_cutlass_kernel(op_args, verbose=False)

    cmd = ["cutlass_profiler"]
    cmd.append("--kernels={}".format(kernel))
    cmd.append("--m={}".format(op_args.m))
    cmd.append("--n={}".format(op_args.n))
    cmd.append("--k={}".format(op_args.k))
    cmd.append("--beta=1")
    cmd.append("--A={}".format(op_args.dtype))
    cmd.append("--B={}".format(op_args.dtype))
    cmd.append("--C={}".format(op_args.dtype))
    cmd.append("--accum={}".format(op_args.dtype))
    lines = subprocess.check_output(cmd, shell=False).decode('utf-8').split("\n")
    header = lines[-3].split(",")
    data = lines[-2].split(",")

    if args.cache_path is not None:
        with open(args.cache_path, 'a') as fp:
            fp.write("{},{},{},{}\n".format(
                data[header.index("Operation")],
                data[header.index("Bytes")],
                data[header.index("Flops")],
                data[header.index("Runtime")]
                ))

# cutlass_tensorop_h1688gemm_256x128_32x2_nn_align8


from django.shortcuts import render
from resumeApp.models import *
from resumeApp.forms import ContactForm
from django.core.mail import EmailMessage
from django.shortcuts import redirect
from django.template import Context
from django.template.loader import get_template
from django.contrib import messages

def home(request):
	jobs = Job.objects.all().order_by('-start_date')
	executive_summary = GeneralInfo.objects.values('executive_summary').get()
	education_list = Education.objects.all().order_by('-year')
	skills = GeneralInfo.objects.values('skills').get()
	context = {
		'jobs': jobs,
		'executive_summary': executive_summary,
		'education_list': education_list,
		'skills': skills,
	}
	return render(request, 'index.html', context)

def contact(request):
	form_class = ContactForm

	if request.method == 'POST':
		form = form_class(data=request.POST)

		if form.is_valid():
			contact_name = request.POST.get(
				'contact_name', '')
			contact_email = request.POST.get(
				'contact_email', '')
			form_content = request.POST.get('content', '')

			# Email the profile with the contact information
			template = get_template('contact_template.txt')
			context = Context({
				'contact_name': contact_name,
				'contact_email': contact_email,
				'form_content': form_content,
			})
			content = template.render(context)

			email = EmailMessage(
				"New Form Submission",
				content,
				"peterdimaria.com" + '',
				['dimaria.p@gmail.com'],
				headers={'Reply-To': contact_email}
			)
			email.send()
			messages.success(request, 'Your message was sent to Peter.')
			return redirect('/contact/')

	return render(request, 'contact.html', {'form': form_class})

def about(request):
	about_copy = GeneralInfo.objects.values('about').get()
	context = {
		'about': about_copy
	}
	return render(request, 'about.html', context)

def privacy_policy(request):
	privacy_policy_copy = GeneralInfo.objects.values('privacy_policy').get()
	context = {
		'privacy_policy': privacy_policy_copy,
	}
	return render(request, 'privacy_policy.html', context)


import os

import affine
from tile2net.logger import logger
import shapely
from geopandas import GeoDataFrame, read_file, sjoin
import pandas as pd
import numpy as np
import rasterio
from shapely.geometry import mapping, shape
import skimage
from affine import Affine

import warnings

warnings.filterwarnings('ignore')

def read_gdf(path):
    """
    Read a GeoDataFrame from a file
    Parameters
    ----------
    path: str
        path to the file

    Returns
    -------
    gdf: GeoDataFrame
    """
    gdf = read_file(path)
    return gdf

def set_gdf_crs(gdf, crs):
    """
    Set the CRS of a GeoDataFrame
    Parameters
    ----------
    gdf: GeoDataFrame
    crs: int
        coordinate reference system

    Returns
    -------
    gdf: GeoDataFrame
    """
    gdf.geometry = gdf.geometry.set_crs(crs)
    return gdf

def change_crs(gdf, crs):
    """
    Change the CRS of a GeoDataFrame
    Parameters
    ----------
    gdf: GeoDataFrame
    crs: int

    Returns
    -------
    gdf: GeoDataFrame
    """
    gdf.geometry = gdf.geometry.to_crs(crs)
    return gdf

def prepare_spindex(gdf: GeoDataFrame) -> GeoDataFrame.sindex:
    """
    Prepare a GeoDataFrame for spatial indexing
    Parameters
    ----------
    gdf: GeoDataFrame

    Returns
    -------
    spatial index of a GeoDataFrame
    """
    return gdf.sindex

def _reduce_geom_precision(geom, precision=2):
    """
    Reduce the precision of a geometry to a given number of decimal places.
    Parameters
    ----------
    geom: shapely.geometry
    precision: int
        number of decimal places to round to

    Returns
    -------
    geom: shapely.geometry
    """
    geojson = mapping(geom)
    geojson['coordinates'] = np.round(
        np.array(geojson['coordinates']),
        precision
    )
    return shape(geojson)

def affine_to_list(affine_obj: affine.Affine):
    """Convert a :class:`affine.Affine` instance to a list for Shapely."""
    return [affine_obj.a, affine_obj.b,
            affine_obj.d, affine_obj.e,
            affine_obj.xoff, affine_obj.yoff]

def list_to_affine(xform_mat: list):
    """Create an Affine from a list or array-formatted [a, b, d, e, xoff, yoff]

    Arguments
    ---------
    xform_mat : `list` or :class:`numpy.array`
        A `list` of values to convert to an affine object.

    Returns
    -------
    aff : :class:`affine.Affine`
        An affine transformation object.
    """
    # first make sure it's not in gdal order
    if len(xform_mat) > 6:
        xform_mat = xform_mat[0:6]
    if rasterio.transform.tastes_like_gdal(xform_mat):
        return Affine.from_gdal(*xform_mat)
    else:
        return Affine(*xform_mat)

def _check_rasterio_im_load(im):
    """Check if `im` is already loaded in; if not, load it in."""
    if isinstance(im, str):
        return rasterio.open(im)
    elif isinstance(im, rasterio.DatasetReader):
        return im
    else:
        raise ValueError(
            "{} is not an accepted image format for rasterio.".format(im)
        )

def _check_skimage_im_load(im):
    """Check if `im` is already loaded in; if not, load it in."""
    if isinstance(im, str):
        return skimage.io.imread(im)
    elif isinstance(im, np.ndarray):
        return im
    else:
        raise ValueError(
            "{} is not an accepted image format for scikit-image.".format(im)
        )

def prepare_class_gdf(polys, class_name) -> GeoDataFrame:
    """
    separates the polygons of each class, given the keyboard (sidewalk, crosswalk, road)
    Args:
        polys (geodataframe): the dataframe containing the polygons of all classes
        class_name(str): the class label, sidewalk, crosswalk, road

    Returns:
        class specific GeoDataFrame in metric projection
    """

    nt = polys[polys.f_type == f'{class_name}'].copy()
    nt.geometry = nt.geometry.to_crs(3857)
    return nt

def prepare_gdf(gdf, **cols):
    """
    Filter a GeoDataFrame based on a set of columns and values
    Parameters
    ----------
    gdf: GeoDataFrame
    cols: dict
        {column_name: value}

    Returns
    -------
    f_gdf: GeoDataFrame
    """
    # TODO: Add other operations like !
    k = list(cols.keys())[0]
    print(f'k, {k}', f'cols {len(cols[k])}')
    if isinstance(cols[k], list):
        f_gdf = gdf[gdf[k].isin(cols[k])]
    else:
        f_gdf = gdf[gdf[k] == cols[k]]
    return f_gdf

def read_dataframe(src_path, geo=True, cols=None):
    """
    Args:
        src_path:
        geo: if True, will create GeoDataFrame
        cols: optional. Name of specific columns to be read
    Returns:
    """
    if geo:
        if cols:
            df = read_file(src_path, usecols=cols)
        else:
            df = read_file(src_path)
    else:
        if cols:
            df = pd.read_csv(src_path, usecols=cols)
        else:
            df = pd.DataFrame(src_path)
    return df

def unary_multi(gdf: GeoDataFrame) -> GeoDataFrame:
    # handles the errors with multipolygon
    loc = ~gdf.is_valid.values
    # logger.warning(f'Number of invalid geometries: {loc.sum()} out of {len(gdf)}')
    count = loc.sum()
    if count:
        logger.warning(f'Number of invalid geometries: {count} out of {len(gdf)}')
    gdf.geometry.loc[loc] = shapely.make_valid(gdf.geometry.loc[loc])
    result = (
        gdf
        # dissolve overlapping geometries
        .dissolve()
        # explode multipart geometries
        .explode()
    )
    return result

def buffer_union(gdf, buff, simp1, simp2):
    """
    buffer and union the polygons in a GeoDataFrame
    Parameters
    ----------
    gdf: GeoDataFrame
    buff: float
        buffer distance
    simp1: float
        simplification tolerance for the buffer
    simp2: float
        simplification tolerance for the union

    Returns
    -------
    gdf_uni: GeoDataFrame
    """
    gdf.geometry = gdf.geometry.buffer(buff, join_style=2, cap_style=3)
    gdf.geometry = gdf.simplify(simp1)
    gdf_uni = unary_multi(gdf)
    gdf_uni.geometry = gdf_uni.geometry.set_crs(3857)
    gdf_uni.geometry = gdf_uni.geometry.simplify(simp2)
    return gdf_uni

def buffer_union_erode(gdf: GeoDataFrame, buff: float, erode: float, simp1: float, simp2: float, simp3: float):
    """ Buffer, union, erode, simplify the polygons in a GeoDataFrame to create elongated polygons
    Parameters
    ----------
    gdf: GeoDataFrame
        Polygon GeoDataFrame
    buff: float
        buffer distance
    erode: float
        erode distance
    simp1: float
        simplification tolerance for the buffer
    simp2: float
        simplification tolerance for the union
    simp3: float
        simplification tolerance for the final simplification
    Returns
    -------
    gdf_uni: GeoDataFrame
        Transformed GeoDataFrame
    """
    gdf_buff = buffer_union(gdf, buff, simp1, simp2)
    gdf_erode = gdf_buff.copy()
    gdf_erode.geometry = gdf_buff.geometry.buffer(erode, join_style=2, cap_style=3)
    gdf_uni = unary_multi(gdf_erode)
    gdf_uni.geometry = gdf_uni.geometry.set_crs(3857)
    gdf_uni.geometry = gdf_uni.geometry.simplify(simp3)
    gdf_uni = gdf_uni[(gdf_uni.geometry.geom_type == 'MultiPolygon') | (gdf_uni.geometry.geom_type == 'Polygon')]

    return gdf_uni

def to_metric(gdf, crs=3857):
    """Converts a GeoDataFrame to metric (3857) coordinate
    Parameters
    ----------
    gdf : GeoDataFrame
        GeoDataFrame of polygons
    crs : int, optional
        the coordinate system to convert to, by default 3857
    Returns
    -------
    GeoDataFrame
        GeoDataFrame of polygons in metric coordinate system
    """
    gdf.to_crs(crs, inplace=True)
    return gdf

def geo2geodf(geo_lst: list) -> GeoDataFrame:
    """
    Converts a list of shapely geometries to a GeoDataFrame
    Parameters
    ----------
    geo_lst: list

    Returns
    -------
    gdf: GeoDataFrame

    """
    gdf = GeoDataFrame(geometry=geo_lst)
    return gdf

def merge_dfs(gdf1: GeoDataFrame, gdf2: GeoDataFrame, crs: int = 4326):
    """
    merges two geodataframes of segmentation results (three classes)
    Parameters
    ----------
    gdf1: GeoDataFrame
    gdf2: GeoDataFrame
    crs: int
        Coordinate system to convert to
    Returns
    -------
    concsw: GeoDataFrame
        Concatenated GeoDataFrames
    """
    if gdf1.crs != gdf2.crs:
        gdf1.to_crs(crs, inplace=True)
        gdf2.to_crs(crs, inplace=True)

    df1sw = prepare_class_gdf(gdf1, 'sidewalk')
    df1cw = prepare_class_gdf(gdf1, 'crosswalk')
    df1rd = prepare_class_gdf(gdf1, 'road')

    df2sw = prepare_class_gdf(gdf2, 'sidewalk')
    df2cw = prepare_class_gdf(gdf2, 'crosswalk')
    df2rd = prepare_class_gdf(gdf2, 'road')

    concsw = pd.concat([df1sw, df2sw])
    conccw = pd.concat([df1cw, df2cw])
    concrd = pd.concat([df1rd, df2rd])

    unionsw = unary_multi(concsw)
    unionsw = unionsw.explode().reset_index(drop=True)
    unionsw.geometry = unionsw.geometry.set_crs(crs)
    unionsw['f_type'] = 'sidewalk'

    unioncw = unary_multi(conccw)

    unioncw.geometry = unioncw.geometry.set_crs(crs)
    unioncw['f_type'] = 'crosswalk'

    unionrd = unary_multi(concrd)

    unionrd.geometry = unionrd.geometry.set_crs(crs)
    unionrd['f_type'] = 'road'

    merged = pd.concat([unionrd, unionsw, unioncw])
    merged.geometry = merged.geometry.set_crs(crs)

    return merged

def create_stats(gdf: GeoDataFrame):
    """ Creates summary statistics of the polygons
    Parameters
    ----------
    gdf: GeoDataFrame

    Returns
    -------
    ss: GeoDataFrame
        summary statistics of the polygons
    cgdf: GeoDataFrame
        copy of the input GeoDataFrame with additional columns of the summary statistics
    """
    cgdf = gdf.copy()
    cgdf['primeter'] = cgdf.length
    cgdf['area'] = cgdf.area
    cgdf['ar_pratio'] = cgdf.area / cgdf.length
    # get the summary statics of the polygons
    ss = cgdf.quantile([0.25, 0.5, 0.75])
    return ss, cgdf

def buff_dfs(gdf: GeoDataFrame):
    """
    union and buffer the polygons of each class separately,
    to create continuous polygons and merge them into one GeoDataFrame.

    Parameters
    ----------
    gdf: GeoDataFrame
        Polygon dataframes with three classes in metric coordinate system
    crs: int

    Returns
    -------
    GeoDataFrame:
        merged GeoDataFrame of the three classes
    """

    gdf.geometry = gdf.simplify(0.2)
    dfsw = prepare_class_gdf(gdf, 'sidewalk')
    dfcw = prepare_class_gdf(gdf, 'crosswalk')
    dfrd = prepare_class_gdf(gdf, 'road')

    buffersw = buffer_union_erode(dfsw, 0.3, -0.3, 0.2, 0.3, 0.3)

    buffersw['f_type'] = 'sidewalk'

    buffercw = buffer_union_erode(dfcw, 0.3, -0.25, 0.2, 0.3, 0.3)

    buffercw['f_type'] = 'crosswalk'

    bufferrd = buffer_union_erode(dfrd, 0.4, -0.4, 0.2, 0.3, 0.3)
    bufferrd['f_type'] = 'road'

    merged = pd.concat([buffercw, buffersw, bufferrd])
    merged.geometry = merged.geometry.set_crs(gdf.crs)

    return merged


import datetime
import json
import logging

import boto3
import custom_connector_sdk.connector.auth as auth
import custom_connector_sdk.connector.configuration as config
import custom_connector_sdk.connector.context as context
import custom_connector_sdk.connector.fields as fields
import custom_connector_sdk.connector.settings as settings
import custom_connector_sdk.lambda_handler.requests as requests
import custom_connector_sdk.lambda_handler.responses as responses
from custom_connector_sdk.lambda_handler.handlers import (
    ConfigurationHandler,
    MetadataHandler,
    RecordHandler,
)
from custom_connector_sdk.lambda_handler.lambda_handler import (
    BaseLambdaConnectorHandler,
)
from custom_connector_sdk.lambda_handler.responses import ErrorCode, ErrorDetails
from google.analytics.data_v1beta import BetaAnalyticsDataClient
from google.analytics.data_v1beta.types import (
    Dimension,
    Metric,
    OrderBy,
    RunRealtimeReportRequest,
)
from google.oauth2 import service_account

LOGGER = logging.getLogger()
LOGGER.setLevel(logging.INFO)

# カスタムコネクタの情報
CONNECTOR_OWNER = "DemoOwner"
CONNECTOR_NAME = "GA4Connector"
CONNECTOR_VERSION = "1.0"

# GA4 への接続情報のキー名（変更不要）
PROPERTY_ID_KEY = "propertyId"
PRIVATE_KEY_KEY = "privateKey"
CLIENT_EMAIL_KEY = "clientEmail"

# スキーマ定義（API 毎に定義）
# 今回は RunRealtimeReport API のみサポートするのでそのエンティティを定義
run_realtime_report_entity = context.Entity(
    entity_identifier="RunRealtimeReportEntity",
    label="RunRealtimeReportEntity",
    has_nested_entities=False,
    description="RunRealtimeReportEntity",
)

class ConnectorDemoConfigurationHandler(ConfigurationHandler):
    def validate_connector_runtime_settings(
        self, request: requests.ValidateConnectorRuntimeSettingsRequest
    ) -> responses.ValidateConnectorRuntimeSettingsResponse:
        print("ConfigurationHandler::validate_connector_runtime_settings")
        return responses.ValidateConnectorRuntimeSettingsResponse(is_success=True)

    def validate_credentials(
        self, request: requests.ValidateCredentialsRequest
    ) -> responses.ValidateCredentialsResponse:
        print("ConfigurationHandler::validate_credentials")
        return responses.ValidateCredentialsResponse(is_success=True)

    # このメソッドが CustomConnector 登録時に呼び出される
    # 実装内容は連携先システムへの認証方法の定義
    def describe_connector_configuration(
        self, request: requests.DescribeConnectorConfigurationRequest
    ) -> responses.DescribeConnectorConfigurationResponse:
        print("ConfigurationHandler::describe_connector_configuration")

        # コネクタの認証方法の設定
        # 今回は GA4 Data API を叩く際に GCP のサービスアカウントを使用するため Custom Auth を設定
        authentication_config = auth.AuthenticationConfig(
            is_custom_auth_supported=True,
            custom_auth_config=[
                auth.CustomAuthConfig(
                    authentication_type="CUSTOM",
                    auth_parameters=[
                        auth.AuthParameter(
                            key=PRIVATE_KEY_KEY,
                            required=True,
                            label="Private Key",
                            description="GCP サービスアカウントの private_key を入力（スペースは \\s に置換）",
                            sensitive_field=True,
                            connector_supplied_values=None,
                        ),
                        auth.AuthParameter(
                            key=CLIENT_EMAIL_KEY,
                            required=True,
                            label="Client Email",
                            description="xxxxxx@xxxx.iam.gserviceaccount.com",
                            sensitive_field=False,
                            connector_supplied_values=None,
                        ),
                    ],
                )
            ],
        )

        property_id_setting = settings.ConnectorRuntimeSetting(
            key=PROPERTY_ID_KEY,
            data_type=settings.ConnectorRuntimeSettingDataType.String,
            required=True,
            label="GA4 Property ID",
            description="GA4 Property ID",
            scope=settings.ConnectorRuntimeSettingScope.CONNECTOR_PROFILE,
        )

        return responses.DescribeConnectorConfigurationResponse(
            is_success=True,
            # カスタムコネクタの所有者
            connector_owner=CONNECTOR_OWNER,
            # カスタムコネクタ名
            connector_name=CONNECTOR_NAME,
            # 認証設定
            authentication_config=authentication_config,
            # このカスタムコネクタのバージョン
            connector_version=CONNECTOR_VERSION,
            # サポートしている GA4 Data API のバージョン
            supported_api_versions=["GoogleAnalyticsDataAPIv1"],
            # カスタムコネクタのモードを設定
            # このカスタムコネクタは送信元にのみ指定できるため, ConnectorModes.SOURCE のみ設定
            connector_modes=[config.ConnectorModes.SOURCE],
            # コネクタの runtime の設定
            # 今回は GA4 のプロパティ ID を設定
            connector_runtime_setting=[property_id_setting],
        )

class ConnectorDemoRecordHandler(RecordHandler):
    def retrieve_data(
        self, request: requests.RetrieveDataRequest
    ) -> responses.RetrieveDataResponse:
        print("RecordHandler::retrieve_data")
        record_list = []
        return responses.RetrieveDataResponse(is_success=True, records=record_list)

    def write_data(
        self, request: requests.WriteDataRequest
    ) -> responses.WriteDataResponse:
        print("RecordHandler::write_data")
        write_record_results = []

        return responses.WriteDataResponse(
            is_success=True, write_record_results=write_record_results
        )

    def query_data(
        self, request: requests.QueryDataRequest
    ) -> responses.QueryDataResponse:
        print("RecordHandler::query_data")

        # コネクタの runtime に GA4 のプロパティ ID が設定されているか検証
        if PROPERTY_ID_KEY not in request.connector_context.connector_runtime_settings:
            error_message = f"{PROPERTY_ID_KEY} should be provided as runtime setting"
            LOGGER.error(f"QueryData request failed with entity: {error_message}")
            return responses.QueryDataResponse(
                is_success=False,
                error_details=ErrorDetails(
                    error_code=ErrorCode.InvalidArgument,
                    error_message=error_message,
                ),
            )

        # 有効なエンティティか検証（ここでは RunRealtimeReportEntity かどうか検証）
        entity_id = request.connector_context.entity_definition.entity.entity_identifier
        if entity_id != run_realtime_report_entity.entity_identifier:
            error_message = f"{entity_id} is not valid entity"
            LOGGER.error(f"QueryData request failed with entity: {error_message}")
            return responses.QueryDataResponse(
                is_success=False,
                error_details=ErrorDetails(
                    error_code=ErrorCode.InvalidArgument,
                    error_message=error_message,
                ),
            )

        # AppFlow の接続作成時に設定した認証情報を Secrets Manager から取得
        secrets = boto3.client("secretsmanager").get_secret_value(
            SecretId=request.connector_context.credentials.secret_arn
        )
        secret_string = json.loads(secrets["SecretString"])

        credentials = service_account.Credentials.from_service_account_info(
            {
                "private_key": secret_string[PRIVATE_KEY_KEY]
                .replace("\\s", " ")
                .replace("\\n", "\n"),
                "client_email": secret_string[CLIENT_EMAIL_KEY],
                "token_uri": "https://oauth2.googleapis.com/token",
            }
        )

        # GA4 Data API を叩く
        client = BetaAnalyticsDataClient(credentials=credentials)

        req = RunRealtimeReportRequest(
            property=f"properties/{request.connector_context.connector_runtime_settings[PROPERTY_ID_KEY]}",
            # 分析項目の指定
            dimensions=[
                Dimension(name="minutesAgo"),
                Dimension(name="country"),
                Dimension(name="city"),
            ],
            # 集計項目の指定
            metrics=[Metric(name="screenPageViews"), Metric(name="activeUsers")],
            # 順序の指定
            order_bys=[
                OrderBy(
                    dimension=OrderBy.DimensionOrderBy(dimension_name="minutesAgo"),
                    desc=False,
                )
            ],
            # 取得する行数 (最大100,000)
            limit=100000,
        )

        res = client.run_realtime_report(req)

        # 秒を切り捨てた現在時刻を取得
        now_floor_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:00")
        now = datetime.datetime.strptime(now_floor_str, "%Y-%m-%d %H:%M:%S")

        # JSON オブジェクトの文字列の配列
        # RunRealtimeReport のエンティティでマッピングしたデータフィールドだけ送信される
        # record_list = [
        #     '{"DateTime": "2022-01-01 00:00:00", "RequestDateTime": "2022-01-01 00:00:00", "ScreenPageViews": 1}',
        #     '{"DateTime": "2022-01-01 00:01:00", "RequestDateTime": "2022-01-01 00:01:00", "ScreenPageViews": 2}',
        # ]
        record_list = []

        for row in res.rows:
            # イベント発生日時の絶対値をイベント発生日時の相対値 (dimensions.minutesAgo) から割り出す
            time = now - datetime.timedelta(minutes=int(row.dimension_values[0].value))

            record_list.append(
                json.dumps(
                    {
                        # レポートをリクエストした時間
                        # * DIMENSION が同じで METRIC が異なるデータは RequestDateTime が新しい方が正確なデータ
                        "RequestDateTime": now.strftime("%Y-%m-%d %H:%M:00"),
                        # [DIMENSION]: イベントの発生日時
                        "DateTime": time.strftime("%Y-%m-%d %H:%M:00"),
                        # [DIMENSION]: イベントを発生させた訪問者の国名
                        "Country": row.dimension_values[1].value,
                        # [DIMENSION]: イベントを発生させた訪問者の町名
                        "City": row.dimension_values[2].value,
                        # [METRIC]: dimensions に対応する訪問者がページを開いた回数
                        "ScreenPageViews": row.metric_values[0].value,
                        # [METRIC]: dimensions に対応するアクティブな訪問者の数
                        "ActiveUsers": row.metric_values[1].value,
                    }
                )
            )

        return responses.QueryDataResponse(is_success=True, records=record_list)

class ConnectorDemoMetadataHandler(MetadataHandler):
    # フロー作成時に呼び出され、スキーマ定義のリストを返す
    def list_entities(
        self, request: requests.ListEntitiesRequest
    ) -> responses.ListEntitiesResponse:
        print("MetadataHandler::list_entities")

        # フロー作成時のスキーマ定義
        # 今回は RunRealtimeReport API のみサポートしているため 1 つのエンティティを設定
        entity_list = [run_realtime_report_entity]

        return responses.ListEntitiesResponse(is_success=True, entities=entity_list)

    # フロー作成のデータフィールドマッピングで呼び出され、フィールドの定義を返す
    def describe_entity(
        self, request: requests.DescribeEntityRequest
    ) -> responses.DescribeEntityResponse:
        print("MetadataHandler::describe_entity")

        date_time_field = context.FieldDefinition(
            field_name="DateTime",
            data_type=fields.FieldDataType.String,
            data_type_label="string",
            label="DateTime",
            description="DateTime",
            default_value="1970-01-01 00:00:00",
            is_primary_key=True,
            # データの送信元に指定する場合に設定するフィールドの型定義
            read_properties=fields.ReadOperationProperty(
                # データ取得時にフィルタとして指定できる項目かどうか
                is_queryable=True,
                # データ取得時に表示されるかどうか
                is_retrievable=True,
                # null 値を許容するかどうか
                is_nullable=False,
                # Date か DateTime 型かどうか
                is_timestamp_field_for_incremental_queries=False,
            ),
            # データの送信先に指定する場合に設定するフィールドの型定義
            write_properties=None,
        )

        request_date_time_field = context.FieldDefinition(
            field_name="RequestDateTime",
            data_type=fields.FieldDataType.String,
            data_type_label="string",
            label="RequestDateTime",
            description="RequestDateTime",
            default_value="1970-01-01 00:00:00",
            is_primary_key=False,
            read_properties=fields.ReadOperationProperty(
                is_queryable=True,
                is_retrievable=True,
                is_nullable=False,
                is_timestamp_field_for_incremental_queries=False,
            ),
            write_properties=None,
        )

        country_field = context.FieldDefinition(
            field_name="Country",
            data_type=fields.FieldDataType.String,
            data_type_label="string",
            label="Country",
            description="Country",
            default_value="Country",
            is_primary_key=False,
            read_properties=fields.ReadOperationProperty(
                is_queryable=True,
                is_retrievable=True,
                is_nullable=False,
                is_timestamp_field_for_incremental_queries=False,
            ),
            write_properties=None,
        )

        city_field = context.FieldDefinition(
            field_name="City",
            data_type=fields.FieldDataType.String,
            data_type_label="string",
            label="City",
            description="City",
            default_value="City",
            is_primary_key=False,
            read_properties=fields.ReadOperationProperty(
                is_queryable=True,
                is_retrievable=True,
                is_nullable=False,
                is_timestamp_field_for_incremental_queries=False,
            ),
            write_properties=None,
        )

        screen_page_views_field = context.FieldDefinition(
            field_name="ScreenPageViews",
            data_type=fields.FieldDataType.Integer,
            data_type_label="int",
            label="ScreenPageViews",
            description="ScreenPageViews",
            default_value="0",
            is_primary_key=False,
            read_properties=fields.ReadOperationProperty(
                is_queryable=True,
                is_retrievable=True,
                is_nullable=False,
                is_timestamp_field_for_incremental_queries=False,
            ),
            write_properties=None,
        )

        active_users_field = context.FieldDefinition(
            field_name="ActiveUsers",
            data_type=fields.FieldDataType.Integer,
            data_type_label="int",
            label="ActiveUsers",
            description="ActiveUsers",
            default_value="0",
            is_primary_key=False,
            read_properties=fields.ReadOperationProperty(
                is_queryable=True,
                is_retrievable=True,
                is_nullable=False,
                is_timestamp_field_for_incremental_queries=False,
            ),
            write_properties=None,
        )

        # 今回は RunRealtimeReport API の DataTime, RequestDatetime, ScreenPageViews を fields に設定
        entity_definition = context.EntityDefinition(
            entity=run_realtime_report_entity,
            fields=[
                date_time_field,
                request_date_time_field,
                country_field,
                city_field,
                screen_page_views_field,
                active_users_field,
            ],
        )

        return responses.DescribeEntityResponse(
            is_success=True, entity_definition=entity_definition
        )

class ConnectorDemoLambdaHandler(BaseLambdaConnectorHandler):
    def __init__(self):
        super().__init__(
            ConnectorDemoMetadataHandler(),
            ConnectorDemoRecordHandler(),
            ConnectorDemoConfigurationHandler(),
        )

def handler(event, context):
    return ConnectorDemoLambdaHandler().lambda_handler(event, context)


import json
import re
import types

import boto3
import botocore.exceptions
import botocore.errorfactory

from .lib import toiter, sip, jsonify
from .secrets import aws_credentials
from .secrets import (
  AWS_DEFAULT_REGION
)

import tenacity

AWS_BATCH_SIZE = 10 # send_message_batch's max batch size is 10

class ClientSideError(Exception):
  pass

retry = tenacity.retry(
  reraise=True,
  stop=tenacity.stop_after_attempt(4),
  wait=tenacity.wait_random_exponential(0.5, 60.0),
  retry=tenacity.retry_if_not_exception_type(ClientSideError),
)

class AWSTaskQueueAPI(object):
  def __init__(self, qurl, region_name=AWS_DEFAULT_REGION, **kwargs):
    """
    qurl: either a queue name (e.g. 'pull_queue') or a url
      like https://sqs.us-east-1.amazonaws.com/DIGITS/wms-pull-queue
    kwargs: Keywords for the underlying boto3.client constructor, other than `service_name`,
      `region_name`, `aws_secret_access_key`, or `aws_access_key_id`.
    """
    if 'region' in kwargs:
      region_name = kwargs.pop('region')

    matches = re.search(r'sqs.([\w\d-]+).amazonaws', qurl)

    if matches is not None:
      region_name, = matches.groups()
      self.qurl = qurl
    else:
      self.qurl = None

    credentials = aws_credentials()

    self.sqs = boto3.client('sqs',
      aws_secret_access_key=credentials.get('AWS_SECRET_ACCESS_KEY'),
      aws_access_key_id=credentials.get('AWS_ACCESS_KEY_ID'),
      region_name=region_name,
      **kwargs,
    )

    if self.qurl is None:
      self.qurl = self._get_qurl(qurl)

    self.batch_size = AWS_BATCH_SIZE

  @retry
  def _get_qurl(self, qurl):
    try:
      return self.sqs.get_queue_url(QueueName=qurl)["QueueUrl"]
    except Exception as err:
      print(f"Failed to fetch queue URL for: {qurl}")
      raise

  @property
  def enqueued(self):
    status = self.status()
    return (
      int(status['ApproximateNumberOfMessages'])
      + int(status['ApproximateNumberOfMessagesNotVisible'])
      + int(status['ApproximateNumberOfMessagesDelayed'])
    )

  @property
  def inserted(self):
    return float('NaN')

  @property
  def completed(self):
    return float('NaN')

  @property
  def leased(self):
    status = self.status()
    return int(status['ApproximateNumberOfMessagesNotVisible'])

  def is_empty(self):
    return self.enqueued == 0

  @retry
  def status(self):
    resp = self.sqs.get_queue_attributes(
      QueueUrl=self.qurl,
      AttributeNames=[
        'ApproximateNumberOfMessages',
        'ApproximateNumberOfMessagesNotVisible',
        'ApproximateNumberOfMessagesDelayed'
      ]
    )
    return resp['Attributes']

  @retry
  def insert(self, tasks, delay_seconds=0):
    tasks = toiter(tasks)

    total = 0
    # send_message_batch's max batch size is 10
    for batch in sip(tasks, self.batch_size):
      if len(batch) == 0:
        break

      entries = [ {
          "Id": str(j),
          "MessageBody": jsonify(task),
          "DelaySeconds": delay_seconds,
        } for j, task in enumerate(batch)
      ]

      try:
        resp = self.sqs.send_message_batch(
          QueueUrl=self.qurl,
          Entries=entries,
        )
      except botocore.exceptions.ClientError as error:
        http_code = error.response['ResponseMetadata']['HTTPStatusCode']
        if 400 <= int(http_code) < 500:
          raise ClientSideError(error)
        else:
          raise error

      total += len(entries)

    return total

  def add_insert_count(self, ct):
    pass

  def rezero(self):
    pass

  @retry
  def renew_lease(self, task, seconds):
    self.sqs.change_message_visibility(
      QueueUrl=self.qurl,
      ReceiptHandle=task.id,
      VisibilityTimeout=seconds,
    )

  def cancel_lease(self, task):
    self.renew_lease(task, 0)

  def release_all(self):
    raise NotImplementedError()

  def lease(self, seconds, num_tasks=1, wait_sec=20):
    if wait_sec is None:
      wait_sec = 20

    resp = self.sqs.receive_message(
      QueueUrl=self.qurl,
      AttributeNames=[
        'SentTimestamp'
      ],
      MaxNumberOfMessages=num_tasks,
      MessageAttributeNames=[
        'All'
      ],
      VisibilityTimeout=seconds,
      WaitTimeSeconds=wait_sec,
    )

    if 'Messages' not in resp:
      return []

    tasks = []
    for msg in resp['Messages']:
      task = json.loads(msg['Body'])
      task['id'] = msg['ReceiptHandle']
      tasks.append(task)
    return tasks

  def delete(self, task):
    if type(task) == str:
      rhandle = task
    else:
      try:
        rhandle = task._id
      except AttributeError:
        rhandle = task['id']

    try:
      self.sqs.delete_message(
        QueueUrl=self.qurl,
        ReceiptHandle=rhandle,
      )
    except botocore.exceptions.ClientError as err:
      pass

    return 1

  def tally(self):
    pass

  def purge(self, native=False):
    # can throw:
    # except botocore.errorfactory.PurgeQueueInProgress:
    if native:
      self.sqs.purge_queue(QueueUrl=self.qurl)
      return

    while self.enqueued:
      # visibility_timeout must be > 0 for delete to work
      tasks = self.lease(num_tasks=10, seconds=10)
      for task in tasks:
        self.delete(task)

  def __iter__(self):
    return iter(self.lease(num_tasks=10, seconds=0))


from django.urls import include, re_path
from django.urls.conf import path

from ratings import views

app_name = 'ratings'

videopatterns = [
    re_path(r'^switch$', views.switch_state, name='switch'),
    re_path(r'^remove$', views.remove_like, name='remove'),
    re_path(r'^dislike$', views.dislike_video, name='dislike'),
    re_path(r'^like$', views.like_video, name='like')
]

urlpatterns = [
    path('video/', include((videopatterns, app_name), namespace='rate_video')),

    # re_path(r'^rate-video', views.rate_video, name='video'),
    re_path(r'^rate-comment', views.rate_comment, name='comment'),
    re_path(r'^rate-reply', views.rate_reply, name='reply'),
]


import pytest
import logging

from helpers import (
    disable_and_delete_resource,
    post_resource,
    get_shared_service_id_by_name,
)
from resources import strings

LOGGER = logging.getLogger(__name__)

@pytest.mark.shared_services
async def test_patch_firewall(admin_token, verify):
    template_name = strings.FIREWALL_SHARED_SERVICE

    patch_payload = {
        "properties": {
            "display_name": "TEST",
            "rule_collections": [
                {
                    "name": "e2e-rule-collection-1",
                    "action": "Allow",
                    "rules": [
                        {
                            "name": "e2e test rule 1",
                            "description": "desc here",
                            "protocols": [{"port": "5555", "type": "Http"}],
                            "target_fqdns": [
                                "one.two.three.microsoft.com",
                                "two.three.microsoft.com"
                            ],
                            "source_addresses": ["172.196.0.0"]
                        }
                    ]
                },
                {
                    "name": "e2e-rule-collection-2",
                    "action": "Allow",
                    "rules": [
                        {
                            "name": "e2e test rule 1",
                            "description": "desc here",
                            "protocols": [{"port": "5556", "type": "Http"}],
                            "target_fqdns": [
                                "one.two.microsoft.com",
                                "two.microsoft.com"
                            ],
                            "source_addresses": ["172.196.0.1"]
                        }
                    ]
                },
                {
                    "name": "e2e-rule-collection-3",
                    "action": "Allow",
                    "priority": 501,
                    "rules": [
                        {
                            "name": "e2e test rule 1",
                            "description": "desc here",
                            "protocols": [{"port": "5557", "type": "Http"}],
                            "target_fqdns": [
                                "one.two.three.microsoft.com.uk"
                            ],
                            "source_addresses": ["172.196.0.2"]
                        }
                    ]
                }
            ],
        },
        "templateName": template_name,
    }

    shared_service_firewall = await get_shared_service_id_by_name(
        template_name, verify, admin_token
    )
    shared_service_path = f'/shared-services/{shared_service_firewall["id"]}'

    await post_resource(
        patch_payload,
        f"/api{shared_service_path}",
        "shared_service",
        admin_token,
        None,
        verify,
        method="PATCH",
    )

shared_service_templates_to_create = [
    (strings.GITEA_SHARED_SERVICE),
    (strings.NEXUS_SHARED_SERVICE),
]

@pytest.mark.shared_services
@pytest.mark.timeout(30 * 60)
@pytest.mark.parametrize("template_name", shared_service_templates_to_create)
async def test_create_shared_service(template_name, admin_token, verify) -> None:
    # Check that the shared service hasn't already been created
    shared_service = await get_shared_service_id_by_name(
        template_name, verify, admin_token
    )
    if shared_service:
        id = shared_service["id"]
        LOGGER.info(
            f"Shared service {template_name} already exists (id {id}), deleting it first..."
        )
        await disable_and_delete_resource(
            f"/api/shared-services/{id}", "shared_service", admin_token, None, verify
        )

    post_payload = {
        "templateName": template_name,
        "properties": {
            "display_name": f"Shared service {template_name}",
            "description": f"{template_name} deployed via e2e tests",
        },
    }

    shared_service_path, _ = await post_resource(
        post_payload,
        "/api/shared-services",
        "shared_service",
        admin_token,
        None,
        verify,
    )

    await disable_and_delete_resource(
        f"/api{shared_service_path}", "shared_service", admin_token, None, verify
    )


from setuptools import setup

package_name = 'cv_basics'

setup(
    name=package_name,
    version='0.0.0',
    packages=[package_name],
    data_files=[
        ('share/ament_index/resource_index/packages',
            ['resource/' + package_name]),
        ('share/' + package_name, ['package.xml']),
    ],
    install_requires=['setuptools'],
    zip_safe=True,
    maintainer='sendol',
    maintainer_email='sendol@todo.todo',
    description='TODO: Package description',
    license='TODO: License declaration',
    tests_require=['pytest'],
    entry_points={
    'console_scripts': [
        'img_publisher = cv_basics.webcam_pub:main',
        'img_subscriber = cv_basics.webcam_sub:main',
        'realsense_publisher = cv_basics.realsense_pub:main',
        'pointcloud_publisher = cv_basics.pointcloud_pub:main',
        'pointcloud_subscriber = cv_basics.pointcloud_sub:main',
        'keyboard_op = cv_basics.keyboard_op:main'
    ],
},
)


import requests
from bs4 import BeautifulSoup
from selenium import webdriver

url = 'https://www.latam.com/es_ec/apps/personas/booking?country=ec&vuelos_fecha_salida_ddmmaaaa=06/12/2020&auAvailability=1&language=es&nadults=1&cabina=Y&ninfants=0&ida_vuelta=ida&fecha1_dia=06&fecha1_anomes=2020-12&from_city1=UIO&flex=1&to_city1=GYE&nchildren=0#/'
agent = {'User-Agent':'Mozila/5.0'}
r = requests.get(url, headers = agent)

print(r.status_code)

s = BeautifulSoup(r.text, 'lxml')
print(s.prettify())

#Instanciar un driver del navegador
options = webdriver.ChromeOptions()
options.add_argument('--incognito')
driver = webdriver.Chrome(executable_path = '../chromedriver', options = options )

#Make the browser load the web page
driver.get(url)

#Extract info
vuelos = driver.find_elements_by_xpath('//li[@class = "flight"]')
print(vuelos)

print('**' * 5)

vuelo = vuelos[0]
vuelo_supp = vuelo
print(vuelo)

duracion_time = vuelo.find_element_by_xpath('.//span[@class = "duration"]/time/span').text
departure_time = vuelo.find_element_by_xpath('.//div[@class = "departure"]/time').get_attribute('datetime')

hora_arrival = vuelo_supp.find_element_by_xpath('.//div[@class = "arrival"]/time').get_attribute('datetime')

print(duracion_time)
print(departure_time)
print(hora_arrival)

driver.close()


import cv2
import numpy as np
from PIL import Image
import math

def get_huan_by_circle(img, circle_center, radius, radius_width):
    black_img = np.zeros((radius_width, int(2 * radius * math.pi), 3), dtype='uint8')
    for row in range(0, black_img.shape[0]):
        for col in range(0, black_img.shape[1]):
            theta = math.pi * 2 / black_img.shape[1] * (col + 1)  # +origin_theta
            rho = radius + row - 1
            p_x = int(circle_center[0] + rho * math.sin(theta) + 0.5) - 1
            p_y = int(circle_center[1] - rho * math.cos(theta) + 0.5) - 1

            black_img[row, col, :] = img[p_y, p_x, :]
    return black_img

img = cv2.imread('img/6_6.bmp')

# 根据霍夫圆得到的圆心以及半径以及截取半径之外厚度得到圆环展开后的矩形图片
img = get_huan_by_circle(img, (809, 867), 450, 50)

# 得到展开后的长宽像素
sum_rows = img.shape[0]
sum_cols = img.shape[1]
print(sum_rows, sum_cols)

for_num = sum_cols // 640 if sum_cols % 640 == False else sum_cols // 640 + 1
# # 图片从中间截断
img_size = 0
for i in range(1, for_num+1):
    locals()[f'part{i}'] = img[0:sum_rows, img_size:i*640]
    img_size += 640

# 新建640大小背景图
imgs = np.zeros((640, 640, 3), np.uint8)

# 反转颜色
imgs[0:,0:] = [255,255,255]
# # # 得到图片居中值
# # left_center = int((640 - (sum_cols // 2))/2-1)
# # up_center = int(640//sum_rows/2-1)
# # # 图片进行居中上下拼接
imgs[0:sum_rows, 0:640] = part1
imgs[sum_rows*2:sum_rows*3, 0:640] = part2
imgs[sum_rows*4:sum_rows*5, 0:640] = part3
imgs[sum_rows*6:sum_rows*7, 0:640] = part4
imgs[sum_rows*8:sum_rows*9, 0:sum_cols-((sum_cols//640)*640)] = part5

# # print(imgs.shape)
#
# cv2.imshow("1", part1)
# cv2.imshow("2", part2)
# cv2.imshow("3", part3)
# cv2.imshow("4", part4)
# cv2.imshow("5", part5)
print(img.shape)
cv2.imshow("6", imgs)
cv2.waitKey(0)


import kodinhenki.wemo.device

from xml.dom.minidom import parseString

import kodinhenki.compat as compat
urllib_request = compat.get_urllib_request()
urlopen = urllib_request.urlopen
urljoin = compat.get_urllib_parse().urljoin

import logging
_debug = logging.debug
_error = logging.error

s_ns = 'urn:Belkin:service-1-0'
soap_ns = 'http://schemas.xmlsoap.org/soap/envelope/'

def _e_children(n):
    return [n for n in n.childNodes if n.nodeType == n.ELEMENT_NODE]
def _unique_e_child(n):
    l = list(_e_children(n))
    assert len(l) == 1
    return l[0]

class Action:
    def __init__(self, service, name):
        self.service = service
        self.name = name
        self.url = urljoin(service.url, service.control_url)
    def __call__(self, **kwargs):
        service = self.service.service_type
        action = self.name
        argl = ['<%s>%s</%s>' % (k, v, k) for (k, v) in kwargs.items()]
        args = ''.join(argl)
        data = '''<?xml version="1.0"?>
<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/">
  <soap:Body>
    <s:%(action)s xmlns:s="%(service)s">
      %(args)s
    </s:%(action)s>
  </soap:Body>
</soap:Envelope>''' % locals()
        headers = {'SOAPAction': '"%s#%s"' % (service, action),
        'Content-Type': 'text/xml'}
        _debug('url:%s - headers:%s - data:%s', self.url, headers, data)
        req = urllib_request.Request(self.url, data.encode('utf-8'), headers)
        try:
            data = urlopen(req, None, 5).read()
        except:
            _error('action call to %s failed' % self.url)
            return
        doc = parseString(data)
        # Doc contains Envelope + Body + X + then inside that, key = value
        bl = doc.getElementsByTagNameNS(soap_ns, 'Body')
        assert len(bl) >= 1
        b = bl[0]

        r = _unique_e_child(b)

        rh = {}
        for e in _e_children(r):
            rh[e.localName] = kodinhenki.wemo.device._get_text_l(e.childNodes)
        return rh

class Service:
    populated = False
    def __init__(self, url, scdp, **kwargs):
        self.url = url
        self.scdp = scdp
        self.__dict__.update(kwargs)
    def populate(self):
        if self.populated: return
        nurl = urljoin(self.url, self.scdp)
        try:
            data = urlopen(nurl, None, 5).read()
        except:
            pass
        self.populated = True
        assert self.scdp
        #_debug('got sdata: %s', data)
        doc = parseString(data)
        assert doc
        _wemo_string = kodinhenki.wemo.device._wemo_string
        for al in doc.getElementsByTagName('actionList'):
            for a in al.getElementsByTagName('action'):
                n = _wemo_string(a, 'name')
                setattr(self, n, Action(self, n))
    def __getattr__(self, x):
        if not self.populated:
            self.populate()
            # Retry default behavior, perhaps we won't wind up here again
            return getattr(self, x)
        else:
            raise AttributeError()

def from_url_object(url, o):
    _wemo_string = kodinhenki.wemo.device._wemo_string
    return Service(url=url,
                   service_type=_wemo_string(o, 'serviceType'),
                   control_url=_wemo_string(o, 'controlURL'),
                   event_sub_url=_wemo_string(o, 'eventSubURL'),
                   scdp=_wemo_string(o, 'SCPDURL'))


# -*- coding: utf-8 -*-
"""
Created on Wed May 10 18:39:34 2023

@author: Abhi
"""

import re
from io import StringIO

import nltk

from pdfminer.high_level import extract_text
from pdfminer.converter import TextConverter
from pdfminer.layout import LAParams
from pdfminer.pdfdocument import PDFDocument
from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
from pdfminer.pdfpage import PDFPage
from pdfminer.pdfparser import PDFParser
from pdfminer.high_level import extract_pages
from pdfminer.layout import LTTextContainer, LTTextBox

from text_utils import TextUtils

from typing import List

class FileUtils:
    '''Contains all the methods related to file operations
    in the context of research papers and citations'''

    # originally ReadFile()
    @classmethod
    def get_words_from_file(cls, file_name: str):
        '''Read the file and return list of words'''
        with open(file_name, 'r') as file:
            return [word.strip() for word in file.readlines()]

    @classmethod
    def read_pdf(cls, file_name: str = None):
        '''Extract text from PDF File'''
        if not isinstance(file_name, str):
            raise TypeError("String must be provided")
        if not file_name:
            raise TypeError("Filename not provided!")
        text = extract_text(file_name)
        text = text.strip().lower()
        text = TextUtils.unicode_to_ascii(text)
        text = TextUtils.remove_unicode(text)
        return text.strip().split('\n\n')

    @classmethod
    def read_pdf_buffered(cls, file_name: str = None):
        '''Extract text from PDF File'''
        if not isinstance(file_name, str):
            raise TypeError("String must be provided")
        if not file_name:
            raise TypeError("Filename not provided!")

        output_string = StringIO()
        with open(file_name, 'rb') as in_file:
            parser = PDFParser(in_file)
            doc = PDFDocument(parser)
            rsrcmgr = PDFResourceManager()
            device = TextConverter(rsrcmgr, output_string, laparams=LAParams())
            interpreter = PDFPageInterpreter(rsrcmgr, device)
            for page in PDFPage.create_pages(doc):
                interpreter.process_page(page)

        return output_string.getvalue()

    @classmethod
    def read_page_text_incremented(cls, file_name: str = None):
        '''Extract text from PDF File page by page'''
        if not isinstance(file_name, str):
            raise TypeError("String must be provided")
        if not file_name:
            raise TypeError("Filename not provided!")

        texts = []
        for page_layout in extract_pages(file_name):
            for element in page_layout:
                if isinstance(element, LTTextContainer):
                    if isinstance(element, LTTextBox):
                        print(f"Textbox index: {element.index}")
                        print("Text content:")
                        print(element.get_text())
                        content = element.get_text().strip()
                        content = TextUtils.unicode_to_ascii(content)
                        content = TextUtils.remove_unicode(content)
                        texts.append(content)

        return texts

    @classmethod
    def save_paragraphs(cls, paragraphs: List[str], output_file: str):
        with open(output_file, 'w', encoding='utf-8') as f:
            for paragraph in paragraphs:
                f.write(paragraph + '\n')

    @classmethod
    def extract_paragraphs(cls, file_name: str) -> List[str]:
        paragraphs = []
        with open(file_name, 'rb') as file:
            resource_manager = PDFResourceManager()
            output_stream = StringIO()
            codec = 'utf-8'
            laparams = LAParams()
            converter = TextConverter(
                resource_manager, output_stream, codec=codec, laparams=laparams)
            interpreter = PDFPageInterpreter(resource_manager, converter)

            for page in PDFPage.get_pages(file, check_extractable=True):
                interpreter.process_page(page)
                extracted_text = output_stream.getvalue()

                # Sanitize the extracted text
                sanitized_text = TextUtils.unicode_to_ascii(extracted_text)
                sanitized_text = re.sub(
                    r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]', ' ', extracted_text)
                sanitized_text = re.sub(r'[^\x20-\x7E\n]', ' ', sanitized_text)
                sanitized_text = re.sub(r'\s+', ' ', sanitized_text)

                # paragraphs.extend(sanitized_text.splitlines())
                for line in sanitized_text.splitlines():
                    sent = nltk.sent_tokenize(line)
                    paragraphs.extend(sent)
                output_stream.truncate(0)
                output_stream.seek(0)

            converter.close()
            output_stream.close()

        return paragraphs


import os
import time
import argparse
import datetime
import numpy as np
import torch
import torch.optim as optim
from matplotlib import pyplot as plt
from torch.autograd import Variable
from torch.utils.data import DataLoader
from Models.actor import DRL4TSP
from Models.tsp.tspActor import NetworkTSP
from Models.tsp.tspPointerNetwork import CombinatorialRLTSP
from Tasks import vrp, tsp
from Models.critc import StateCritic, StateCriticTSP
from Tasks.tsp import TSPDataset
from utils.plots import plot_metrics, plot_models_metrics
torch.backends.cudnn.benchmark = True
torch.backends.cudnn.enabled = False
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print('Detected device {}'.format(device))

metrics_per_model = []

def test(data_loader, actor, reward_fn, render_fn=None, save_dir='.', num_plot=5):
    """Used to monitor progress on a validation set & optionally plot solution."""

    actor.eval()

    if not os.path.exists(save_dir):
        os.makedirs(save_dir)

    rewards = []
    for batch_idx, static in enumerate(data_loader):
        static = static.to(device)
        with torch.no_grad():
            tour_indices, _ = actor.forward(static, static)

        reward = reward_fn(static, tour_indices).mean().item()
        rewards.append(reward)

        if render_fn is not None and batch_idx < num_plot:
            name = 'batch%d_%2.4f.png' % (batch_idx, reward)
            path = os.path.join(save_dir, name)
            render_fn(static, tour_indices, path)

    actor.train()
    return np.mean(rewards)

def validate(data_loader, actor, reward_fn, render_fn=None, save_dir='.', num_plot=5):
    """Used to monitor progress on a validation set & optionally plot solution."""

    actor.eval()

    if not os.path.exists(save_dir):
        os.makedirs(save_dir)

    rewards = []
    for batch_idx, batch in enumerate(data_loader):

        static = batch

        static = static.to(device)

        with torch.no_grad():
            reward, probs, actions, actions_idxs = actor.forward(static)

        #reward = reward_fn(static, tour_indices).mean().item()
        rewards.append(reward)

        # if render_fn is not None and batch_idx < num_plot:
        #     name = 'batch%d_%2.4f.png' % (batch_idx, reward)
        #     path = os.path.join(save_dir, name)
        #     render_fn(static, actions_idxs, path)

    actor.train()
    return np.mean(rewards)

def train(actor, critic, task, num_nodes, train_data, valid_data, reward_fn,
          render_fn, batch_size, actor_lr, critic_lr, max_grad_norm, experiment,
          **kwargs):
    """Constructs the main actor & critic networks, and performs all training."""

    now = '%s' % datetime.datetime.now().time()
    now = now.replace(':', '_')
    save_dir = os.path.join(task, '%d' % num_nodes, now)

    print('Starting training')

    checkpoint_dir = os.path.join(save_dir, 'checkpoints')
    if not os.path.exists(checkpoint_dir):
        os.makedirs(checkpoint_dir)

    actor_optim = optim.Adam(actor.parameters(), lr=actor_lr)
    critic_optim = optim.Adam(critic.parameters(), lr=critic_lr)

    lr_decay = 1.0
    # lr_scheduler = optim.lr_scheduler.LambdaLR(actor_optim, lambda epoch: lr_decay ** epoch)

    train_loader = DataLoader(train_data, batch_size, True, num_workers=0)
    valid_loader = DataLoader(valid_data, batch_size, False, num_workers=0)

    best_reward = np.inf

    losses_per_epoch = []
    rewards_per_epoch = []
    critic_losses_per_epoch = []
    critic_rewards_per_epoch = []
    distances_per_epoch = []

    for epoch in range(epochs):

        actor.train()
        critic.train()

        times, losses, rewards, critic_rewards, distances, critic_losses = [], [], [], [], [], []

        epoch_start = time.time()
        beta = 0.9

        for batch_idx, batch in enumerate(train_loader):

            static = Variable(batch)

            static = static.to(device)

            # Full forward pass through the dataset
            reward, probs, actions, actions_idxs =actor(static)

            if batch_idx == 0:
                critic_exp_mvg_avg = reward.mean()
            else:
                critic_exp_mvg_avg = (critic_exp_mvg_avg * beta) + ((1. - beta) * reward.mean())

            # Sum the log probabilities for each city in the tour
            #reward = reward_fn(static, tour_indices)

            # Query the critic for an estimate of the reward
            #critic_est = critic(static).view(-1)
            logprobs = 0
            for prob in probs:
                logprob = torch.log(prob)
                logprobs += logprob
            logprobs[logprobs < -1000] = 0.

            advantage = (reward - critic_exp_mvg_avg)#- critic_est)

            reinforce = advantage  * logprobs
            actor_loss = reinforce.mean() # torch.mean(advantage.detach() * tour_logp)

            #critic_loss = torch.mean(advantage ** 2)

            actor_optim.zero_grad()
            actor_loss.backward()
            #torch.nn.utils.clip_grad_norm_(actor.parameters(), max_grad_norm)
            actor_optim.step()

            critic_exp_mvg_avg = critic_exp_mvg_avg.detach()
            # critic_optim.zero_grad()
            # critic_loss.backward()
            # torch.nn.utils.clip_grad_norm_(critic.parameters(), max_grad_norm)
            # critic_optim.step()
            #
            # critic_rewards.append(torch.mean(critic_est.detach()).item())
            # critic_losses.append(torch.mean(critic_loss.detach()).item())

            # actor reward is the absolute log probabilities of the chosen actions multiplied
            # by the advantage
            actor_reward = torch.mean(torch.abs(reinforce))

            rewards.append(actor_reward.item())

            losses.append(torch.mean(actor_loss.detach()).item())
            distances.append(torch.mean(reward).item())

        # epoch finished
        # lr_scheduler.step()

        # print(f'Learning rate for epoch {epoch}={lr_scheduler.get_lr()}')
        mean_critic_reward = np.mean(critic_rewards)
        mean_critic_loss = np.mean(critic_losses)
        mean_loss = np.mean(losses)
        mean_reward = np.mean(rewards)
        mean_distance = np.mean(distances)

        losses_per_epoch.append(mean_loss)
        rewards_per_epoch.append(mean_reward)
        distances_per_epoch.append(mean_distance)

        critic_losses_per_epoch.append(mean_critic_loss)
        critic_rewards_per_epoch.append(mean_critic_reward)

        # Save the weights
        epoch_dir = os.path.join(checkpoint_dir, '%s' % epoch)
        if not os.path.exists(epoch_dir):
            os.makedirs(epoch_dir)

        save_path = os.path.join(epoch_dir, 'actor.pt')
        torch.save(actor.state_dict(), save_path)

        save_path = os.path.join(epoch_dir, 'critic.pt')
        torch.save(critic.state_dict(), save_path)

        # Save rendering of validation set tours
        valid_dir = os.path.join(save_dir, '%s' % epoch)

        mean_valid = validate(valid_loader, actor, reward_fn, render_fn,
                              valid_dir, num_plot=5)

        print(
            f'Mean epoch {epoch}/{epochs}: loss={mean_loss}, reward={mean_reward}, distance={mean_distance}, val reward={mean_valid}')

    plt.clf()

    metrics = [distances_per_epoch,
               critic_losses_per_epoch,
               critic_rewards_per_epoch,
               losses_per_epoch,
               rewards_per_epoch]

    labels = ["Distances", "Critic loss", "Critic estimate", "Actor loss", "Actor reward"]

    metrics_per_model.append(metrics)
    plot_metrics(metrics,
                 labels,
                 experiment,
                 labels,
                 title="CVRP",
                 xlabel="Epochs")
    plt.clf()
    plt.plot(losses_per_epoch)
    plt.savefig(f"losses{experiment}.png")
    plt.clf()

    plt.plot(rewards_per_epoch)
    plt.savefig(f"rewards{experiment}.png")
    plt.clf()

    plt.plot(distances_per_epoch)
    plt.savefig(f"distances{experiment}.png")
    plt.clf()

    plt.plot(critic_losses_per_epoch)
    plt.savefig(f"critic_losses{experiment}.png")
    plt.clf()

    plt.plot(critic_rewards_per_epoch)
    plt.savefig(f"critic_rewards{experiment}.png")
    plt.clf()

    torch.save(actor.state_dict(), f'actor{experiment_name}.pt')
    torch.save(critic.state_dict(), f'critic{experiment_name}.pt')

def train_vrp(args):

    print('Starting TSP training')

    STATIC_SIZE = 2  # (x, y)
    train_data = TSPDataset(20, train_size,  args.seed + 1)
    valid_data = TSPDataset(20, args.valid_size,  args.seed + 2)

    # actor = NetworkTSP(STATIC_SIZE,
    #                 args.hidden_size,
    #                 None,
    #                 train_data.update_mask,
    #                 args.embedding,
    #                 args.num_layers,
    #                 args.dropout).to(device)

    embedding_size = 128
    hidden_size = 128
    n_glimpses = 1
    tanh_exploration = 10
    use_tanh = True
    batch_size = 128
    beta = 0.9
    max_grad_norm = 2.

    """Δημιουργούμε τα μοντέλα που θα εκπαιδεύσουμε."""

    actor = CombinatorialRLTSP(
        embedding_size,
        hidden_size,
        20,
        n_glimpses,
        tanh_exploration,
        use_tanh,
        tsp.reward,
        embedding="Graph",
        attention="Dot")

    # actor = CombinatorialRLTSP(
    #     args.hidden_size,
    #     args.hidden_size,
    #     20,
    #     1,
    #     10,
    #     True,
    #     tsp.reward,
    #     attention="Dot")

    print('Actor: {} '.format(actor))

    critic = StateCriticTSP(STATIC_SIZE, args.hidden_size, args.embedding).to(device)

    print('Critic: {}'.format(critic))

    kwargs = vars(args)
    kwargs['train_data'] = train_data
    kwargs['valid_data'] = valid_data
    kwargs['reward_fn'] = vrp.reward
    kwargs['render_fn'] = vrp.render

    if not args.test:
        train(actor, critic, **kwargs)

    # test_data = TSPDataset( args.train_size, num_nodes, args.seed + 2)
    #
    # test_dir = 'test'

    # testing
    # finding variation
    test_distances = []

    # for i in range(10):
    #     test_loader = DataLoader(test_data, args.batch_size, False, num_workers=0)
    #     avg_tour_length = test(test_loader, actor, vrp.reward, vrp.render, test_dir, num_plot=5)
    #     print('Average tour length: ', avg_tour_length)
    #
    #     test_distances.append(avg_tour_length)
    #
    #     variance = np.var(test_distances)
    #     plt.clf()
    #     plt.plot(test_distances)
    #     plt.title(f"Test distances, variance: {variance}")
    #     plt.savefig("testDistances.png")

if __name__ == '__main__':
    train_size = 15_000#_000
    num_nodess = [20]#, 50, 100]

    bss = [100]
    # bss = [256] -> oles kales oi metrikes ektos distances # [128, 256, 512]
    epochs = 20#0
    emb = ['conv']  # , 'lin', 'graph']
    critic_lr = 5e-1
    actor_lr = 1e-4

    for num_nodes_idx in range(len(num_nodess)):
        num_nodes = num_nodess[num_nodes_idx]
        bs = bss[0]
        embedding = emb[0]
        parser = argparse.ArgumentParser(description='Combinatorial Optimization')
        parser.add_argument('--seed', default=12345, type=int)
        parser.add_argument('--checkpoint', default=None)
        parser.add_argument('--test', action='store_true', default=False)
        parser.add_argument('--task', default='vrp')
        parser.add_argument('--nodes', dest='num_nodes', default=num_nodes, type=int)
        parser.add_argument('--actor_lr', default=actor_lr, type=float)
        parser.add_argument('--critic_lr', default=critic_lr, type=float)
        parser.add_argument('--max_grad_norm', default=2., type=float)
        parser.add_argument('--batch_size', default=bs, type=int)  # 256
        parser.add_argument('--hidden', dest='hidden_size', default=128, type=int)
        parser.add_argument('--dropout', default=0.1, type=float)
        parser.add_argument('--layers', dest='num_layers', default=1, type=int)
        parser.add_argument('--train-size', default=train_size, type=int)  # 0000
        parser.add_argument('--valid-size', default=1000, type=int)
        parser.add_argument('--embedding', default=embedding)

        experiment_name = f'TAHN_EXP_SP_CRITIC_AVG_nodes_{num_nodes}_{embedding}size{train_size}_bs{bs}_actLr{actor_lr}_criLr{critic_lr}'
        parser.add_argument('--experiment', default=experiment_name)

        args = parser.parse_args()

        train_vrp(args)

    # metric_labels = ["Distances", "Critic loss", "Critic reward", "Loss", "Reward"]
    # # labels = ['Conv embedding', 'Linear embedding', 'Graph embedding']
    # labels = ['10 nodes', '50 nodes', '100 nodes']
    #
    # plot_models_metrics(metrics_per_model[0],
    #                     metrics_per_model[1],
    #                     metrics_per_model[2],
    #                     labels,
    #                     metric_labels,
    #                     experiment_name)
    #
    # k = len(metrics_per_model)
    #
    # for i in range(k):
    #     print(metrics_per_model[i])


"""init

Revision ID: 86800f0880ef
Revises:
Create Date: 2021-10-02 18:42:00.975844

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '86800f0880ef'
down_revision = None
branch_labels = None
depends_on = None

def upgrade():
    op.create_table(
        'user',
        sa.Column('id', sa.Integer, primary_key=True),
        sa.Column('username', sa.String, unique=True),
        sa.Column('email', sa.String, unique=True),
        sa.Column('password', sa.Text, nullable=True),
        sa.Column('is_staff', sa.String, default=False),
        sa.Column('is_active', sa.String, default=True),
    )

    op.create_table(
        'orders',
        sa.Column('id', sa.Integer, primary_key=True),
        sa.Column('quantity', sa.Integer, nullable=False),
        sa.Column('order_status', sa.String, default="PENDING"),
        sa.Column('pizza_size', sa.String, default="SMALL"),
        sa.Column('user_id', sa.Integer, nullable=False),
        sa.ForeignKeyConstraint(['user_id'], ['user.id'],),
    )

def downgrade():
    op.drop_table('orders')
    op.drop_table('user')


import os
import sys
from datetime import date

from unittest import TestCase

from dataclasses import dataclass
from typing import Dict, Union, List

from ktpocr import KTPExtractor, KTPIdentity

verbose = True if '-v' in sys.argv else False

ReportType = Dict[str, Union[float, bool]]

@dataclass
class TestResult:
    filepath: str
    accuracy: float
    result: dict
    identity: KTPIdentity

    def print_result(self):
        if verbose:
            self.draw_table()
        accuracy =f'{self.accuracy * 100:.0f} %'
        print(f'\n{os.path.basename(self.filepath):20}: {accuracy}', end='')

    def draw_table(self):
        buffer = ''

        for title, result in self.result.items():
            value = getattr(self.identity, title)
            if isinstance(value, date):
                value = str(value)
            row = f"{title:15}: {result * 100:>7.2f}%\t{value}"
            buffer += row + '\n'

        max_width = max([len(line) for line in buffer.split('\n')])

        buffer = "=" * max_width + '\n' + buffer.strip()
        print(buffer)

class TestAccuracy(TestCase):
    def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)
        self.test_results: List[TestResult] = []

    def get_test_image_path(self, name: str) -> str:
        project_path = os.path.dirname(os.path.dirname(__file__))
        return os.path.join(project_path, 'test', 'resources', name)

    def get_accuracy(self, report: ReportType) -> float:
        total_score = 0.0
        for _, value in report.items():
            if isinstance(value, float):
                total_score += value
            else:
                total_score += (1.0 if value == True else 0.0)
        avg = total_score/len(report.items())
        return avg

    def doCleanups(self) -> None:
        for result in self.test_results:
            result.print_result()

    def test_clean_image(self):
        ktp_truth = KTPIdentity(
            number="3171234567890123",
            name="MIRA SETIAWAN",
            birth_place="JAKARTA",
            birth_date=date(1986, 2, 18),
            sex="PEREMPUAN",
            full_address="JL. PASTI CEPAT A7/66",
            neigborhood="007/008",
            district="PEGADUNGAN",
            sub_district="KALIDERES BANGKAHULU",
            religion="ISLAM",
            marital="KAWIN",
            job="PEGAWAI SWASTA",
            nationality="WNI",
            valid_date=date(2017, 2, 22)
        )
        img_path = self.get_test_image_path('image_clean.jpeg')

        extractor = KTPExtractor(img_path, treshold=150)
        identity = extractor.extract()

        result = identity.compare(ktp_truth)
        accuracy = self.get_accuracy(result)

        self.test_results.append(TestResult(img_path, accuracy, result, identity))
        self.assertGreaterEqual(accuracy, .8)

    def test_ktp1(self):
        ktp_truth = KTPIdentity(
            number="177104240190002",
            name="GALANG RAKE PRATAMA",
            birth_place="BENGKULU",
            birth_date=date(1993, 1, 4),
            sex="LAKI-LAKI",
            full_address="JLUNIB PERMA III NO 35 PERUMNAS UNIB",
            neigborhood="015/003",
            district="PEMATANG GUBERNUR",
            sub_district="MUARA BANGKAHULU",
            religion="ISLAM",
            marital="BELUM KAWIN",
            job="PELAJAR/MAHASISWA",
            nationality="WNI",
            valid_date=date(2017, 1, 24)
        )
        img_path = self.get_test_image_path('ktp1.jpeg')

        extractor = KTPExtractor(img_path, treshold=150)
        identity = extractor.extract()

        result = identity.compare(ktp_truth)
        accuracy = self.get_accuracy(result)

        self.test_results.append(TestResult(img_path, accuracy, result, identity))
        self.assertGreaterEqual(accuracy, .8)

    def test_ktp2(self):
        ktp_truth = KTPIdentity(
            number="3217061804870007",
            name="ARIEF WIJAYA PUTRA",
            birth_place="BANDUNG",
            birth_date=date(1987, 4, 18),
            sex="LAKI-LAKI",
            full_address="JL. AMIR MAHMUD GG. SIRNAGALIH NO.62",
            neigborhood="005/006",
            district="CIBABAT",
            sub_district="CIMAHI UTARA",
            religion="ISLAM",
            marital="BELUM KAWIN",
            job="PELAJAR/MAHASISWA",
            nationality="WNI",
            valid_date="SEUMUR HIDUP"
        )
        img_path = self.get_test_image_path('ktp2.jpeg')

        extractor = KTPExtractor(img_path, treshold=150)
        identity = extractor.extract()
        result = identity.compare(ktp_truth)
        accuracy = self.get_accuracy(result)

        self.test_results.append(TestResult(img_path, accuracy, result, identity))
        self.assertGreaterEqual(accuracy, .8)

    def test_ktp3(self):
        ktp_truth = KTPIdentity(
            number="3523160606800003",
            name="MAFTUCHIN",
            birth_place="TUBAN",
            birth_date=date(1980, 6, 6),
            sex="LAKI-LAKI",
            full_address="JL. AMIR MAHMUD GG. SIRNAGALIH NO.62",
            neigborhood="001/002",
            district="SUGIHARJO",
            sub_district="TUBAN",
            religion="ISLAM",
            marital="KAWIN",
            job="WIRASWASTA",
            nationality="WNI",
            valid_date=date(2017, 6, 6)
        )
        img_path = self.get_test_image_path('ktp3.jpeg')

        extractor = KTPExtractor(img_path, treshold=150)
        identity = extractor.extract()

        result = identity.compare(ktp_truth)
        accuracy = self.get_accuracy(result)

        self.test_results.append(TestResult(img_path, accuracy, result, identity))
        self.assertGreaterEqual(accuracy, .8)

    def test_ktp5(self):
        ktp_truth = KTPIdentity(
            number="1204050503670001",
            name="EDO FURNAMA",
            birth_place="NIAS",
            birth_date=date(1967, 3, 5),
            sex="LAKI-LAKI",
            full_address="DUSUN II HILIHAMBAWA",
            neigborhood="001/003",
            district="HILIGODU TANASOE",
            sub_district="HILIDUHO",
            religion="KATHOLIK",
            marital="CERAI HIDUP",
            job="WARTAWAN",
            nationality="WNI",
            valid_date=date(2018, 3, 5)
        )
        img_path = self.get_test_image_path('ktp5.jpeg')

        extractor = KTPExtractor(img_path, treshold=150)
        identity = extractor.extract()

        result = identity.compare(ktp_truth)
        accuracy = self.get_accuracy(result)

        self.test_results.append(TestResult(img_path, accuracy, result, identity))
        self.assertGreaterEqual(accuracy, .8)

    def test_ktp6(self):
        ktp_truth = KTPIdentity(
            number="3216061812590006",
            name="WIDIARSO",
            birth_place="PEMALANG",
            birth_date=date(1959, 12, 18),
            sex="LAKI-LAKI",
            full_address="SKU JL.SUMATRA BLOK B78/15",
            neigborhood="003/004",
            district="MEKARSARI",
            sub_district="TAMBUN SELATAN",
            religion="KATHOLIK",
            marital="KAWIN",
            job="KARYAWAN SWASTA",
            nationality="WNI",
            valid_date=date(2018, 12, 18)
        )
        img_path = self.get_test_image_path('ktp6.jpeg')

        extractor = KTPExtractor(img_path, treshold=150)
        identity = extractor.extract()

        result = identity.compare(ktp_truth)
        accuracy = self.get_accuracy(result)

        self.test_results.append(TestResult(img_path, accuracy, result, identity))
        self.assertGreaterEqual(accuracy, .8)

    def test_ktp7(self):
        ktp_truth = KTPIdentity(
            number="3471140209790001",
            name="RIYANTO. SE",
            birth_place="GROBOGAN",
            birth_date=date(1976, 9, 2),
            sex="LAKI-LAKI",
            full_address="PRM PURI DOMAS D-3, SEMPU",
            neigborhood="001/024",
            district="WEDOMARTINI",
            sub_district="NGEMPLAK",
            religion="ISLAM",
            marital="KAWIN",
            job="PEDAGANG",
            nationality="WNI",
            valid_date=date(2017, 9, 2)
        )
        img_path = self.get_test_image_path('ktp7.jpeg')

        extractor = KTPExtractor(img_path, treshold=150)
        identity = extractor.extract()

        result = identity.compare(ktp_truth)
        accuracy = self.get_accuracy(result)
        extractor._save_all_image('ktp7_all.jpg')

        self.test_results.append(TestResult(img_path, accuracy, result, identity))
        self.assertGreaterEqual(accuracy, .8)


from collections import namedtuple

#: Wrapper class for values that don't have to be quoted.
atom = namedtuple("atom", ("value",))

#: The set of characters that must be escaped before being sent as
#: Redis strings.
escapes = {
    ord(b"\0"): rb"\x00",
    ord(b"\n"): rb"\n",
    ord(b"\r"): rb"\r",
    ord(b"\\"): rb"\\",
    ord(b'"'): rb'\"',
}

def serialize(x):
    """Serialize `x` so that it can safely be sent to Redis.

    Parameters:
      x(object): The value to serialize.

    Returns:
      bytes: The serialized value.
    """
    if isinstance(x, atom):
        return x.value
    elif isinstance(x, bytes):
        return quote(x)
    elif isinstance(x, str):
        return quote(x.encode("utf-8"))
    elif isinstance(x, (float, int)):
        return str(x).encode("ascii")
    else:
        return serialize(str(x))

def quote(bs):
    return b'"' + bytes(escape(bs)) + b'"'

def escape(bs):
    for c in bs:
        if c in escapes:
            yield from escapes[c]
        else:
            yield c