from django.http import HttpResponse
from django.template import loader
from django.conf import settings
from django.contrib.sessions.models import Session

from easy_thumbnails.files import get_thumbnailer

from rest_framework.permissions import AllowAny
from rest_framework.views import APIView
from rest_framework.generics import ListCreateAPIView
from rest_framework.response import Response as RestResponse
from rest_framework.generics import RetrieveUpdateDestroyAPIView

from shop.models.customer import CustomerModel

from dshop.models import ProductVariableVariant, ProductDefault
from dshop.transition import quotation_new_notification
from .models import dmQuotation, dmQuotationItem
from .serializers import dmQuotationSerializer, dmQuotationItemSerializer

class dmQuotationCartMergeAPI(APIView):

    def get(self, request, *args, **kwargs):
        cookie = request.GET.get('cookie', None)
        session = Session.objects.filter(
            session_key=request.session.session_key
        )
        customer = None
        if session.count() > 0:
            session = session[0].get_decoded()
            user_id = session.get("_auth_user_id")
            if user_id:
                customer = CustomerModel.objects.get(user__id=user_id)

        if cookie and customer:
            cookie_quotation = dmQuotation.objects.filter(
                cookie=cookie,
                status=1,
                customer=None
            )
            for quot in cookie_quotation:
                quot.customer = customer
                quot.save()
            check_quotation = dmQuotation.objects.filter(
                customer=customer, status=1
            )
            if check_quotation.count() > 1:
                first_quotation = check_quotation.order_by('id').first()
                check_quotation = check_quotation.exclude(
                    id=first_quotation.id
                )
                for quot in check_quotation:
                    for item in dmQuotationItem.objects.filter(quotation=quot):
                        item.quotation = first_quotation
                        item.save()
                    quot.delete()
        return HttpResponse('Ok')

class dmQuotationCartCreateAPI(APIView):

    def post(self, request, *args, **kwargs):    # noqa: C901
        variant = request.GET.get('variant', None)
        product = request.GET.get('product', None)
        quantity = request.GET.get('quantity', '')
        cookie = request.GET.get('cookie', '')
        try:
            if product is not None:
                product_obj = ProductDefault.objects.get(product_code=product)
            elif variant is not None:
                product_obj = ProductVariableVariant.objects.get(product_code=variant)   # noqa
        except Exception as e:
            print(e)
            return RestResponse({"valid": False})

        customer = None

        if not request.user.is_anonymous:
            customer = CustomerModel.objects.get(user=request.user)

        if not customer:
            session = Session.objects.filter(
                session_key=request.session.session_key
            )
            if session.count() > 0:
                session = session[0].get_decoded()
                user_id = session.get('_auth_user_id')
                if user_id:
                    customer = CustomerModel.objects.get(user__id=user_id)

        # Check for Quotation
        quotation = []
        if customer:
            quotation = dmQuotation.objects.filter(
                customer=customer,
                status=1
            ).last()
        if not quotation:
            quotation = dmQuotation.objects.filter(
                cookie=cookie,
                status=1
            ).last()

        if not quotation:
            # To generate Quotation Number
            existing_q = dmQuotation.objects.all().order_by('-id')
            if existing_q:
                number = existing_q.first().number
                number = int(number) + 1
            else:
                number = "00001"
            if len(str(number)) < 5:
                num = '0'
                for i in range(1, 5-len(str(number))):
                    num = str(num) + '0'
                number = num + str(number)
            quotation = dmQuotation.objects.create(
                cookie=cookie,
                customer=customer,
                number=number
            )
        else:
            if customer:
                quotation.customer = customer
                quotation.save()
                '''if cookie:
                    cookie_quotation = dmQuotation.objects.filter(
                        cookie=cookie,
                        status=1,
                        customer=None
                    )
                    for quot in cookie_quotation:
                        quot.customer = customer
                        quot.save()
            check_quotation = dmQuotation.objects.filter(
                customer=customer, status=1
            )
            if check_quotation.count() > 1:
                first_quotation = check_quotation.order_by('id').first()
                check_quotation = check_quotation.exclude(
                    id=first_quotation.id
                )
                for quot in check_quotation:
                    for item in dmQuotationItem.objects.filter(quotation=quot):
                        item.quotation = first_quotation
                        item.save()
                    quot.delete()
                quotation = first_quotation'''

        if product is not None:
            quotation_items = dmQuotationItem.objects.filter(
                quotation=quotation,
                product_code=product_obj.product_code
            )
        if variant is not None:
            quotation_items = dmQuotationItem.objects.filter(
                quotation=quotation,
                variant_code=product_obj.product_code
            )
        if quotation_items:
            # Update Item
            quotation_items[0].quantity += int(quantity)
            quotation_items[0].save()
        else:
            # Create Item
            data = {
                'quotation': quotation,
                'quantity': quantity,
            }
            if variant is not None:
                attributes = ",".join([attr.value for attr in product_obj.attribute.all()])  # noqa
                data['product_type'] = 2
                data['variant_code'] = product_obj.product_code
                data['variant_attribute'] = attributes
                data['product_code'] = product_obj.product.square_id
                data['product_name'] = product_obj.product.product_name
            else:
                attributes = ''
                data['product_type'] = 1
                data['product_code'] = product_obj.product_code
                data['product_name'] = product_obj.product_name
            dmQuotationItem.objects.create(**data)
        return RestResponse({"valid": True})

class dmQuotationListCreateAPI(ListCreateAPIView):

    serializer_class = dmQuotationSerializer
    queryset = dmQuotation.objects.all()
    permission_classes = [AllowAny, ]

    def get_queryset(self):
        if not self.request.user.is_anonymous:
            user = self.request.user
            queryset = self.queryset.filter(customer__user=user)
            return queryset
        return dmQuotation.objects.none()

class dmQuotationRetrieve(RetrieveUpdateDestroyAPIView):

    serializer_class = dmQuotationSerializer
    permission_classes = [AllowAny, ]
    lookup_field = 'pk'
    # queryset = dmQuotation.objects.all()

    def get_queryset(self):
        cust = CustomerModel.objects.get(user=self.request.user)
        query = dmQuotation.objects.filter(
            customer=cust
        )
        return query

    def get(self, request, *args, **kwargs):    # noqa: C901
        cookie = request.GET.get('cookie', None)
        quotation = None
        if not request.user.is_anonymous:
            quotation = dmQuotation.objects.filter(
                pk=kwargs['pk'],
                customer=request.user.customer
            ).last()
        elif cookie is not None:
            quotation = dmQuotation.objects.filter(
                pk=kwargs['pk'],
                cookie=cookie,
                status=1
            ).last()
        else:
            session = Session.objects.filter(
                session_key=request.session.session_key
            ).last()
            if session and session.count() > 0:
                session = session[0].get_decoded()
                user_id = session.get('_auth_user_id')
                if user_id:
                    customer = CustomerModel.objects.get(user__id=user_id)
                    quotation = dmQuotation.objects.filter(
                        pk=kwargs['pk'],
                        customer__user=customer,
                        status=1
                    ).last()
        if quotation is None:
            context = {'quotations': [], 'count': 0}
        else:
            quot = {
                'id': quotation.id,
                'number': quotation.number,
                'status': dmQuotation.CHOICE_STATUS[int(quotation.status)-1][0],  # noqa
                'created_at': quotation.created_at,
                'updated_at': quotation.updated_at
            }
            items = []
            for item in dmQuotationItem.objects.filter(
                quotation=quotation
            ).order_by("product_name"):
                if item.product_type == 1:
                    current_item = ProductDefault.objects.filter(
                        product_code=item.product_code
                    ).first()
                    if current_item.main_image:
                        current_item_image = current_item.main_image
                    elif current_item.images.first():
                        current_item_image = current_item.images.first()
                    else:
                        current_item_image = None
                    if current_item_image is not None:
                        try:
                            current_image = get_thumbnailer(
                                current_item_image
                            ).get_thumbnail({
                                'size': (80, 80),
                                'upscale': True,
                                'background': "#ffffff"
                            }).url
                        except Exception:
                            current_image = None
                    else:
                        current_image = None
                    itm = {
                        'id': item.id,
                        'product_name': item.product_name,
                        'product_code': item.product_code,
                        'product_url': current_item.get_absolute_url(),
                        'product_image': current_image,
                        'variant_attribute': None,
                        'quantity': item.quantity,
                        'price': (
                            item.quantity * item.unit_price
                        ) if item.unit_price else None
                    }
                elif item.product_type == 2:
                    # ===---
                    current_item = ProductVariableVariant.objects.filter(
                        product_code=item.variant_code
                    ).first()
                    if current_item.product.main_image:
                        current_item_image = current_item.product.main_image
                    elif current_item.product.images.first():
                        current_item_image = current_item.product.images.first()  # noqa
                    else:
                        current_item_image = None
                    if current_item_image is not None:
                        try:
                            current_image = get_thumbnailer(
                                current_item_image
                            ).get_thumbnail({
                                'size': (80, 80),
                                'upscale': True,
                                'background': "#ffffff"
                            }).url
                        except Exception:
                            current_image = None
                    else:
                        current_image = None
                    # ===---
                    itm = {
                        'id': item.id,
                        'product_name': item.product_name,
                        'product_code': item.variant_code,
                        'product_url': current_item.product.get_absolute_url(),
                        'product_image': current_image,
                        'variant_attribute': item.variant_attribute,
                        'quantity': item.quantity,
                        'price': (
                            item.quantity * item.unit_price
                        ) if item.unit_price else None
                    }
                items.append(itm)
            quot["items"] = items
            context = {"quotation": quot}
        return RestResponse(context)

    def patch(self, request, *args, **kwargs):
        status = request.data.get("status", None)
        email = request.data.get("customer", None)
        if status is not None and email is not None:
            customer = CustomerModel.objects.filter(
                email=email
            ).last()
            if customer and status == 2:
                quotation = dmQuotation.objects.get(id=kwargs['pk'])
                quotation.customer = customer
                quotation.status = 2
                quotation.save()
                quotation_new_notification(quotation)
                return RestResponse({"valid": True})
        return RestResponse({"valid": False})

class dmQuotationItemListCreateAPI(ListCreateAPIView):

    serializer_class = dmQuotationItemSerializer
    permission_classes = [AllowAny, ]
    queryset = dmQuotationItem.objects.all()

class dmQuotationItemRetrieve(RetrieveUpdateDestroyAPIView):

    serializer_class = dmQuotationItemSerializer
    permission_classes = [AllowAny, ]
    lookup_field = 'pk'
    queryset = dmQuotationItem.objects.all()

def dmQuotationPage(request):
    cookie = request.GET.get('cookie', '')
    template = loader.get_template(
        "theme/{}/pages/quotation.html".format(settings.THEME_SLUG)
    )

    if request.user:
        quotations = dmQuotation.objects.filter(customer__user=request.user)
    elif cookie:
        quotations = dmQuotation.objects.filter(cookie=cookie)
    else:
        return HttpResponse(template.render({}, request))

    if not quotations:
        context = {'quotations': [], 'count': 0}
    else:
        data = []
        for quotation in quotations:
            quot = {
                'id': quotation.id,
                'number': quotation.number,
                'status': dmQuotation.CHOICE_STATUS[int(quotation.status)-1][1],  # noqa
                'created_at': quotation.created_at,
                'updated_at': quotation.updated_at
            }
            items = []
            for item in dmQuotationItem.objects.filter(quotation=quotation):
                itm = {
                    'id': item.id,
                    'product_name': item.product_name,
                    'product_code': item.product_code,
                    'variant_code': item.variant_code,
                    'variant_attribute': item.variant_attribute,
                    'quantity': item.quantity,
                }
                items.append(itm)
            quot["items"] = items
            data.append(quot)
        context = {'quotations': data, 'count': len(data)}
    return HttpResponse(template.render(context, request))

class dmQuotationCurrent(APIView):
    def get(self, request, *args, **kwargs):    # noqa: C901
        cookie = request.GET.get('cookie', None)
        quotation = None
        if not request.user.is_anonymous:
            quotation = dmQuotation.objects.filter(
                customer__user=request.user,
                status=1
            ).last()
        elif cookie is not None:
            quotation = dmQuotation.objects.filter(
                cookie=cookie,
                status=1
            ).last()
        else:
            session = Session.objects.filter(
                session_key=request.session.session_key
            )
            if session:
                session = session[0].get_decoded()
                user_id = session['_auth_user_id']
                customer = CustomerModel.objects.get(user__id=user_id)
                quotation = dmQuotation.objects.filter(
                    customer__user=customer,
                    status=1
                ).last()
        if quotation is None:
            context = {'quotations': [], 'count': 0}
        else:
            quot = {
                'id': quotation.id,
                'number': quotation.number,
                'status': dmQuotation.CHOICE_STATUS[int(quotation.status)-1][1],  # noqa
                'created_at': quotation.created_at,
                'updated_at': quotation.updated_at
            }
            items = []
            for item in dmQuotationItem.objects.filter(
                quotation=quotation
            ).order_by("product_name"):
                if item.product_type == 1:
                    current_item = ProductDefault.objects.filter(
                        product_code=item.product_code
                    ).first()
                    if current_item.main_image:
                        current_item_image = current_item.main_image
                    elif current_item.images.first():
                        current_item_image = current_item.images.first()
                    else:
                        current_item_image = None
                    if current_item_image is not None:
                        try:
                            current_image = get_thumbnailer(
                                current_item_image
                            ).get_thumbnail({
                                'size': (80, 80),
                                'upscale': True,
                                'background': "#ffffff"
                            }).url
                        except Exception:
                            current_image = None
                    else:
                        current_image = None
                    itm = {
                        'id': item.id,
                        'product_name': item.product_name,
                        'product_code': item.product_code,
                        'product_url': current_item.get_absolute_url(),
                        'product_image': current_image,
                        'variant_attribute': None,
                        'quantity': item.quantity,
                    }
                elif item.product_type == 2:
                    # ===---
                    current_item = ProductVariableVariant.objects.filter(
                        product_code=item.variant_code
                    ).first()
                    if current_item.product.main_image:
                        current_item_image = current_item.product.main_image
                    elif current_item.product.images.first():
                        current_item_image = current_item.product.images.first()  # noqa
                    else:
                        current_item_image = None
                    if current_item_image is not None:
                        try:
                            current_image = get_thumbnailer(
                                current_item_image
                            ).get_thumbnail({
                                'size': (80, 80),
                                'upscale': True,
                                'background': "#ffffff"
                            }).url
                        except Exception:
                            current_image = None
                    else:
                        current_image = None
                    # ===---
                    itm = {
                        'id': item.id,
                        'product_name': item.product_name,
                        'product_code': item.variant_code,
                        'product_url': current_item.product.get_absolute_url(),
                        'product_image': current_image,
                        'variant_attribute': item.variant_attribute,
                        'quantity': item.quantity,
                    }
                items.append(itm)
            quot["items"] = items
            context = {"quotation": quot}
        return RestResponse(context)


import os

from pytest import mark, fixture, param

from consts.stx import HEAT_SCENARIO_PATH, FlavorSpec, GuestImages, VMStatus
from consts.proj_vars import ProjVar
from consts.timeout import VMTimeout
from utils.tis_log import LOG
from keywords import nova_helper, vm_helper, heat_helper, network_helper, host_helper, system_helper

VM_SCALE_STACK = 'NestedAutoScale'

def __launch_vm_scale_stack():
    stack_name = VM_SCALE_STACK
    template_name = '{}.yaml'.format(stack_name)
    image = GuestImages.DEFAULT['guest']
    high_val = 50
    template_path = os.path.join(ProjVar.get_var('USER_FILE_DIR'), HEAT_SCENARIO_PATH, template_name)
    key_pair = vm_helper.get_default_keypair()
    net_id = network_helper.get_mgmt_net_id()
    network = network_helper.get_net_name_from_id(net_id=net_id)

    flavor_id = nova_helper.create_flavor(name='heartbeat', cleanup='module')[1]
    extra_specs = {FlavorSpec.GUEST_HEARTBEAT: 'True'}
    nova_helper.set_flavor(flavor=flavor_id, **extra_specs)

    params = {'FLAVOR': flavor_id, 'KEYPAIR': key_pair, 'IMAGE': image, 'NETWORK': network, 'HIGH_VALUE': high_val}
    LOG.fixture_step("Create Heat Stack for auto scalable vms using template {}".format(template_name))
    heat_helper.create_stack(stack_name=stack_name, template=template_path, parameters=params, fail_ok=False,
                             cleanup='module')

    LOG.fixture_step("Verify first VM is created via heat stack for auto scaling")
    vm_id = vm_helper.get_vm_id_from_name(vm_name=stack_name, strict=False)

    return stack_name, vm_id

def __delete_vm_scale_stack():
    stack_name = VM_SCALE_STACK
    LOG.fixture_step("Delete heat stack{}".format(stack_name))
    heat_helper.delete_stack(stack=stack_name)

    LOG.fixture_step("Check heat vms are all deleted")
    heat_vms = vm_helper.get_vms(strict=False, name=stack_name)
    assert not heat_vms, "Heat vms still exist on system after heat stack deletion"

@fixture(scope='module', autouse=True)
def delete_vm_scaling_stack(request):
    """
        Create heat stack using NestedAutoScale.yaml for vm scaling
            - Verify heat stack is created successfully
            - Verify heat resources are created
    """

    def delete_stack():
        __delete_vm_scale_stack()
    request.addfinalizer(delete_stack)

@fixture(scope='function')
def vm_scaling_stack():
    stack_name = VM_SCALE_STACK
    heat_vms = vm_helper.get_vms(strict=False, name=stack_name)
    if len(heat_vms) == 1:
        return stack_name, heat_vms[0]

    if heat_vms:
        __delete_vm_scale_stack()

    stack_name, vm_id = __launch_vm_scale_stack()
    return stack_name, vm_id

@mark.parametrize('actions', [
    param('live_migrate-cold_migrate-swact-host_reboot', marks=mark.p2),
])
def test_heat_vm_scale_after_actions(vm_scaling_stack, actions):

    """
    Test VM auto scaling with swact:
        Create heat stack for auto scaling using NestedAutoScale.yaml,  swact and perform vm scale up and down.

    Test Steps:
        - Create a heat stack for auto scaling vm ()
        - Verify heat stack is created successfully
        - Verify heat resources are created
        - live migrate the vm if not sx
        - cold migrate the vm if not sx
        - swact if not sx
        - reboot -f vm host
        - trigger auto scale by boosting cpu usage in the vm (using dd)
        - verify it scale up to the max number of vms (3)
        - trigger scale down by killing dd in the vm
        - verify the vm scale down to min number (1)
        - Delete Heat stack and verify resource deletion
    """
    stack_name, vm_id = vm_scaling_stack
    vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm_id)

    if not system_helper.is_aio_simplex():
        actions = actions.split('-')
        if "swact" in actions:
            LOG.tc_step("Swact before scale in/out")
            host_helper.swact_host()

        if "live_migrate" in actions:
            LOG.tc_step("live migrate vm before scale in/out")
            vm_helper.live_migrate_vm(vm_id)

        if "cold_migrate" in actions:
            LOG.tc_step("cold migrate vm before scale in/out")
            vm_helper.cold_migrate_vm(vm_id)

    if "host_reboot" in actions:
        if system_helper.is_aio_simplex():
            host_helper.reboot_hosts('controller-0')
            vm_helper.wait_for_vm_status(vm_id, status=VMStatus.ACTIVE, timeout=600, check_interval=10, fail_ok=False)
            vm_helper.wait_for_vm_pingable_from_natbox(vm_id, timeout=VMTimeout.DHCP_RETRY)
        else:
            LOG.tc_step("evacuate vm before scale in/out")
            vm_host = vm_helper.get_vm_host(vm_id=vm_id)
            vm_helper.evacuate_vms(host=vm_host, vms_to_check=vm_id)

    LOG.tc_step("Wait for {} vms to auto scale out to {} after running dd in vm(s)".format(stack_name, 3))
    vm_helper.wait_for_auto_vm_scale_out(stack_name, expt_max=3)

    LOG.tc_step("Wait for {} vms to auto scale in to {} after killing dd processes in vms".format(stack_name, 1))
    vm_helper.wait_for_auto_vm_scale_in(stack_name, expt_min=1)


from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from motor.motor_asyncio import AsyncIOMotorClient
from beanie import init_beanie

from v1.api.database import model
from v1.api.database.settings import MongoDBSettings, origins
from v1.api.database.model import Query

app = FastAPI()

# Set up CORS

app.add_middleware(
    CORSMiddleware,
    allow_origins=origins,
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)
client = AsyncIOMotorClient(MongoDBSettings.database_url)
database = client[MongoDBSettings.database_name]

async def init_models():
    await init_beanie(
        database=database,
        document_models=(
            Query,
            # add additional models here
        ),
    )

@app.on_event("startup")
async def on_startup():
    await init_models()
    print("Initializing App")

@app.on_event("shutdown")
async def on_shutdown():
    client.close()
    print("DataBase Connection Failed")


import os
from absl import app
from absl import logging
from absl import flags
from learner import *
from bandit import BernoulliBandit
from utils import draw, write_to_file

FLAGS = flags.FLAGS

# Flags for general settings
flags.DEFINE_string('out', 'data.out', 'file for generated data')
flags.DEFINE_string('fig', 'figure.png', 'file for generated figure')
flags.DEFINE_boolean('novar', True, 'do not show std in the output figure')
flags.DEFINE_boolean('rm', False, 'remove previously generated data')
flags.DEFINE_boolean('minimax', False, 'compute minimax regret based on given arms')
flags.DEFINE_float('prob', 0.2, 'the true probability of the first arm')
flags.DEFINE_integer('T', 1000, 'time horizon')
flags.DEFINE_integer('trials', 100, 'total number of trials')
flags.DEFINE_integer('freq', 50, 'frenquency to report the intermediate regrets')

# Flags for hyper-parameters
flags.DEFINE_float('eps', 1, 'parameter epsilon for epsilon greedy algorithm')
flags.DEFINE_float('C', 1, 'parameter C for explore-then-commit algorithm')
flags.DEFINE_float('alpha', 0.5, 'parameter alpha for UCB algorithm')

def main(argv):
    del argv

    if FLAGS.rm:
        os.remove(FLAGS.out)
    else:
        if FLAGS.out in os.listdir('./'):
            logging.fatal(('%s is not empty. Make sure you have'
                           ' archived previously generated data. '
                           'Try --rm flag which will automatically'
                           ' delete previous data.') % FLAGS.out)

    # # If you want to generate reproducing result, uncomment the line below
    np.random.seed(200)

    # Extract all flags of parameters for later use
    trials = FLAGS.trials
    freq = FLAGS.freq
    T = FLAGS.T
    eps = FLAGS.eps
    C = FLAGS.C
    alpha = FLAGS.alpha
    prob = FLAGS.prob

    # Policies to be compared
    policies = [Greedy(), EpsGreedy(eps), ExploreThenCommit(C), BerUCB(alpha), TS()]
    # policies = [UCB(alpha)]

    # Get true distribution of Bernoulli arms
    mus = [(prob, 1 - prob)]
    if FLAGS.minimax:
        mus = [(0.4, 0.6), (0.3, 0.7), (0.2, 0.8), (0.25, 0.75), (0.35, 0.65)]

    # The main loop
    for policy in policies:
        logging.info('run policy %s' % policy.name)
        for trial in range(trials):
            if trial % 50 == 0:
                logging.info('trial: %d' % trial)

            minimax_regret = dict()

            for (mu_1, mu_2) in mus:
                bernoulli_bandit = BernoulliBandit(mu_1, mu_2)
                agg_regret = dict()

                # The reason to exclude ExploreThenCommit is that it's not progressive.
                # Progressive means process of small T is nested in the process of large T
                if policy.name == 'ExploreThenCommit':
                    for comp_t in range(0, T + 1):
                        if comp_t % freq == 0:
                            if comp_t == 0:
                                agg_regret[comp_t] = 0
                            else:
                                # initialization
                                bernoulli_bandit.init()
                                policy.init(comp_t)
                                rewards = 0
                                for t in range(1, comp_t + 1):
                                    action = policy.choice(t)
                                    reward = bernoulli_bandit.pull_arm(action)
                                    policy.update(reward, action)
                                    rewards += reward
                                agg_regret[comp_t] = bernoulli_bandit.regret(rewards)
                else:
                    # initialization
                    bernoulli_bandit.init()
                    policy.init()
                    rewards = 0
                    for t in range(0, T + 1):
                        if t > 0:
                            action = policy.choice(t)
                            reward = bernoulli_bandit.pull_arm(action)
                            policy.update(reward, action)
                            rewards += reward
                        if t % freq == 0:
                            agg_regret[t] = bernoulli_bandit.regret(rewards)

                for t in agg_regret:
                    minimax_regret[t] = max(minimax_regret.get(t, 0), agg_regret[t])

            # output one trial result of into the output file
            write_to_file(dict({policy.name: minimax_regret}))

    # Generate the final figure
    draw()

# The main entry
if __name__ == '__main__':
    app.run(main)


from keras.layers import *
from sklearn.model_selection import StratifiedKFold, train_test_split
from keras.preprocessing.sequence import pad_sequences
from keras import backend as K
from keras.models import Model, Sequential, load_model
from keras.losses import mean_squared_error as mse_loss
from data_manipulation import *
from keras import optimizers
from keras.optimizers import RMSprop, Adam
from keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler
import numpy as np
import pandas as pd
import random
import gc
import pickle
from tqdm import tqdm
from sklearn.metrics import mean_squared_error
import tensorflow as tf
from keras import backend as K

K.set_learning_phase(1) #set learning phase

def train_model(keras_model:Model, X_t, y_train, batch_size, epochs, X_v, y_valid, fold, meter=None,
                patience=3, model_dir=cfg.MLP_dir, schedule=cfg.learning_rate_scheduler):
    early_stopping = EarlyStopping(patience=patience, verbose=1)
    if not meter:
        model_checkpoint = ModelCheckpoint(model_dir + "model_".format(meter) + str(fold) + ".hdf5",
                                           save_best_only=True, verbose=1, monitor='val_root_mean_squared_error',
                                           mode='min')
    else:
        model_checkpoint = ModelCheckpoint(model_dir + "meter_{}_model_".format(meter) + str(fold) + ".hdf5",
                                           save_best_only=True, verbose=1, monitor='val_root_mean_squared_error',
                                           mode='min')
    learning_rate_schedule = LearningRateScheduler(schedule=schedule)

    _ = keras_model.fit(X_t, y_train, batch_size=batch_size, epochs=epochs,
                           validation_data=(X_v, y_valid), verbose=1,
                           callbacks=[learning_rate_schedule, early_stopping, model_checkpoint])
    if not meter:
        keras_model = load_model(model_dir + "model_".format(meter) + str(fold) + ".hdf5",
                                 custom_objects={'root_mean_squared_error': root_mean_squared_error})
    else:
        keras_model = load_model(model_dir + "meter_{}_model_".format(meter) + str(fold) + ".hdf5",
                                 custom_objects={'root_mean_squared_error': root_mean_squared_error})
    return keras_model

def get_keras_data(df):
    X = {col: np.array(df[col]) for col in df.columns}
    return X

def get_keras_data_RMLP(df, feature_names):
    X = {feat: df[:, :, col_id] for col_id, feat in zip(range(df.shape[-1]), feature_names)}
    return X

def preprocess_data_for_RMLP(X_train:pd.DataFrame, RMLP=False, test_signal=False):
    if not RMLP:
        categoricals = ["site_id", "building_id", "primary_use", "hour", "weekday", "meter", "month", "dayofyear"]

    elif test_signal:
        categoricals = ["row_id", "site_id", "building_id", "primary_use", "hour", "weekday", "meter", "month", "dayofyear", "year"]

    else:
        categoricals = ["site_id", "building_id", "primary_use", "hour", "weekday", "meter", "month",
                        "dayofyear", "year"]

    numericals = ["square_feet", "age", "air_temperature", "cloud_coverage",
                  "dew_temperature", 'building_min', 'building_max', 'sea_level_pressure']

    feat_cols = categoricals + numericals
    X = X_train[feat_cols]
    if test_signal:
        X['row_id'] += 1  # bias for test row- to solve padded seq problem
    return X

def transform_to_3_dim(X):
    columns_shape = X.shape[1]
    X['single_input'] = X.apply(tuple, axis=1).apply(list)  # organizes the series in a list
    X['single_input'] = X['single_input'].apply(lambda x: [list(x)]) # organizes the list in a list, for concatenation effect when using cumsum
    X['hourly_samples'] = X['single_input'].cumsum()  # concatenate the series
    ans = np.asarray(X.hourly_samples.values[-1])  # we take the last sample which holds all the samples concatenated
    if np.shape(ans)[0] != 24:
        ans = np.reshape(ans, (1, ans.shape[0], ans.shape[-1]))
        ans = pad_sequences(ans, 24, dtype='float32').tolist()
    ans = np.reshape(ans, (-1, 24, columns_shape))
    return ans

def process_batch(X:pd.DataFrame, Y=None):
    X_time_series_batch, Y_batch = [], []
    if Y is not None:
        X['meter_reading'] = Y.values
    feat_columns = X.shape[1] - 1  # -1 is because of year
    for year in list(X['year'].unique()):
        print("Adjusting for year {}".format(year))
        for day_in_year in tqdm(list(X['dayofyear'].unique())):
            criteria = (X['dayofyear'] == day_in_year) & (X['year'] == year)
            X_to_criterion = X.loc[criteria]
            X_to_criterion.drop('year', inplace=True, axis='columns')
            sorted_by_hour = X_to_criterion.groupby(['building_id', 'meter']).\
                apply(lambda x: transform_to_3_dim(x.sort_values(["hour"])))
            sorted_by_hour = sorted_by_hour.values.tolist()
            padded_sequences_to_whole_day = np.reshape(sorted_by_hour, (-1, 24, feat_columns))

            if Y is not None:
                X_tmp = padded_sequences_to_whole_day[:, :, :-1]  # features
                y_tmp = padded_sequences_to_whole_day[:, :, -1]  # meter_Reading
                Y_batch.append(y_tmp)
            else:
                X_tmp = padded_sequences_to_whole_day  # features
            X_time_series_batch.append(X_tmp)
    if Y is not None:
        return X_time_series_batch, Y_batch
    else:
        return X_time_series_batch

def root_mean_squared_error(y_true, y_pred):
    return K.sqrt(K.mean(K.square(y_pred - y_true), axis=0))

# def get_batch_from_dataframe(X_train:pd.DataFrame, Y_train:pd.DataFrame):
#     min_dayyear = X_train['dayofyear'].min()  # usually 0
#     max_dayyear = X_train['dayofyear'].max()  # usually 364
#     day_of_year_batch_values = list(random.sample(range(min_dayyear, max_dayyear), cfg.batch_size))
#     # take only the samples that correspond to the dayofyear we sampled
#     batch_criteria = np.isin(X_train['dayofyear'].values, day_of_year_batch_values)
#     X_batch = X_train[batch_criteria]
#     indices = list(X_batch.index)
#     Y_batch = Y_train.loc[indices]
#     X_batch_timeseries_by_hour, Y_batch_timeseries_by_hour = process_batch(X_batch, Y_batch) # get batch of samples and organize them for LSTM
#     return X_batch_timeseries_by_hour, Y_batch_timeseries_by_hour

def load_RMLP():
    print("------------- Loading RMLP models -------------")
    models = []
    for i in range(cfg.MLP_folds):
        keras_model = load_model(cfg.RMLP_dir + "model_" + str(i) + ".hdf5",
                                 custom_objects={'root_mean_squared_error': root_mean_squared_error})
        models.append(keras_model)
    return models

def RMLP(features, drop_rate=0.2, metrics=root_mean_squared_error, loss='mse', lr=0.001):
    # # Inputs
    # # cat = ['site_id','building_id','meter','primary_use','hour','weekday','dayofyear','month']
    # inputs_dic_index = {}
    # inputs = []
    # for i, feat in enumerate(features):
    #     shape = [24]
    #     tmp_in = Input(shape=shape, name=feat)
    #     inputs.append(tmp_in)
    #     inputs_dic_index[feat] = i
    # # Embeddings layers
    # emb_site_id = Embedding(16, 2, input_length=24)(inputs[inputs_dic_index['site_id']])
    # # 15 site's multiplied by 6 of primary use (there is 5 primary use
    # # that form most of the samples : https://www.kaggle.com/angelapuc/model-lgbm-of-final-ds-notebook)
    # emb_building_id = Embedding(1449, 90, input_length=24)(inputs[inputs_dic_index['building_id']])
    # emb_meter = Embedding(4, 3, input_length=24)(inputs[inputs_dic_index['meter']]) # steam, hot water versus electricity versus cool water
    # emb_primary_use = Embedding(16, 6, input_length=24)(inputs[inputs_dic_index['primary_use']])  # see link above
    # emb_hour = Embedding(24, 3, input_length=24)(inputs[inputs_dic_index['hour']])  # morning, noon and evening
    # emb_weekday = Embedding(7, 2, input_length=24)(inputs[inputs_dic_index['weekday']]) # weekend and not weekend
    # emb_yearday = Embedding(365, 2, input_length=24)(inputs[inputs_dic_index['dayofyear']])  # holidya
    # emb_month = Embedding(12, 4, input_length=24)(inputs[inputs_dic_index['month']])
    # expand = Lambda(lambda x: tf.expand_dims(x, axis=-1))
    # main_l = concatenate([
    #        emb_site_id
    #      , emb_building_id
    #      , emb_meter
    #      , emb_primary_use
    #      , emb_hour
    #      , emb_weekday
    #      , emb_month
    #      , emb_yearday
    #      , expand(inputs[inputs_dic_index['square_feet']])
    #      , expand(inputs[inputs_dic_index['age']])
    #      , expand(inputs[inputs_dic_index['air_temperature']])
    #      , expand(inputs[inputs_dic_index['cloud_coverage']])
    #      , expand(inputs[inputs_dic_index['dew_temperature']])
    #      , expand(inputs[inputs_dic_index['building_min']])
    #      , expand(inputs[inputs_dic_index['building_max']])
    #      , expand(inputs[inputs_dic_index['sea_level_pressure']])
    # ])
    # lstm_layer = LSTM(cfg.lstm_units, return_sequences=True, input_shape=(None, 120))(main_l) # calculation of the rightmost shape of main_l
    # lstm_layer_post_batch = TimeDistributed(BatchNormalization(), input_shape=(24, cfg.lstm_units))(lstm_layer)
    # dense_1 = TimeDistributed(Dense(32, activation='relu'))(lstm_layer_post_batch)
    # dense_1_post_drop = TimeDistributed(Dropout(drop_rate))(dense_1)
    # dense_1_post_drop = TimeDistributed(BatchNormalization())(dense_1_post_drop)
    # prediction_seq = TimeDistributed(Dense(1))(dense_1_post_drop)
    # model = Model(inputs, prediction_seq)
    # model.compile(optimizer=Adam(lr=lr),
    #               loss=mse_loss,
    #               metrics=[root_mean_squared_error])
    model = Sequential()
    # model.add(LSTM(cfg.lstm_units, return_sequences=True, input_shape=(None, features.shape[-1])))
    model.add(LSTM(cfg.lstm_units, return_sequences=True, input_shape=(None, features.shape[-1])))
    model.add(BatchNormalization())
    model.add(Dense(32, activation='relu'))
    model.add(Dropout(drop_rate))
    model.add(BatchNormalization())
    model.add(Dense(1))
    model.compile(optimizer=Adam(lr=lr), loss=loss, metrics=[metrics])
    return model

def evaluate_RMLP(X_test_df, first=False):
    if first:
        print("------------ Creating Time Series data ------------")
        X_test = process_batch(X_test_df)
        with open(cfg.ready_dir + 'X_test_timeseries.pkl', 'wb') as fout:
            pickle.dump(X_test, fout)
    else:
        print("------------ Loading Time Series data ------------")
        with open(cfg.ready_dir + 'X_test_timeseries.pkl', 'rb') as fout:
            X_test = pickle.load(fout)
    X_test = np.vstack(X_test)
    X_test_feat = X_test[:, :, 1:]
    row_col = X_test[:, :, 0].astype(np.int32)  # row id is the first feature
    max_row_id_df = X_test_df.row_id.max()
    max_row_id_timeseries = np.max(row_col)
    assert max_row_id_df == max_row_id_timeseries, "max row of dataframe is not the same as timeseries version"
    print("------------- Predicting Test -------------")
    models = load_RMLP()
    res = pd.read_csv(cfg.data_dir + "/sample_submission.csv")
    res.set_index('row_id', drop=False, inplace=True)
    i = 0
    step_size = 50000
    for _ in tqdm(range(int(np.ceil(X_test_feat.shape[0] / step_size)))):
        for_prediction = X_test_feat[i:min(i + step_size, X_test_feat.shape[0]), :, :]
        indices_for_res = row_col[i:min(i + step_size, X_test_feat.shape[0]), :] - 1  # correct the bias
        tmp = np.expm1(sum([model.predict(for_prediction) for model in models]) / cfg.MLP_folds)
        tmp = tmp[indices_for_res >= 0, :]  # take only the valid rows
        indices_for_res = indices_for_res[indices_for_res >= 0]
        res.loc[indices_for_res, 'meter_reading'] = np.squeeze(tmp)
        i += step_size
    res.loc[res['meter_reading'] < 0, 'meter_reading'] = 0
    res.to_csv('submission_aggregated_RMLP.csv', index=False)

def plotting_train_RMLP_prediction(X_df, Y_df):
    model_name = "RMLP"
    models = load_RMLP()
    print("------------ Loading Time Series data ------------")
    with open(cfg.ready_dir + 'X_timeseries.pkl', 'rb') as fout:
        X = pickle.load(fout)
    with open(cfg.ready_dir + 'Y_timeseries.pkl', 'rb') as fout:
        Y = pickle.load(fout)
    X = np.vstack(X)
    Y = np.expand_dims(np.vstack(Y), axis=-1)
    X = X[:100000, :, :]
    Y = Y[:100000, :]
    print("--------- predicting train---------\n")
    ground_y = Y.ravel()  # create flattened array
    i = 0
    res = np.zeros_like(Y)
    step_size = 50000
    for _ in tqdm(range(int(np.ceil(X.shape[0] / step_size)))):
        for_prediction = X[i:min(i + step_size, X.shape[0]), :, :]
        res[i:min(i + step_size, X.shape[0]), :, :] = sum([model.predict(for_prediction) for model in models]) / cfg.MLP_folds
        i += step_size
    res = res.ravel()
    print("RMSE score : \t {}".format(mean_squared_error(ground_y, res)))
    res[res < 0] = 0
    fig, axes = plt.subplots(1, 1)
    sns.distplot(res, label='pred', ax=axes)
    sns.distplot(ground_y, label='ground_truth', ax=axes)
    axes.set_title("pred_Vs_GT_{}".format(model_name))
    axes.legend()
    fig.show()
    fig.savefig("{}_dist_train.png".format(model_name))

def train_models_using_RMLP(X_df, Y_df, first=False):
    epochs = 250
    models = []
    if first:
        print("------------ Creating Time Series data ------------")
        X, Y = process_batch(X_df.copy(), Y_df.copy())
        with open(cfg.ready_dir + 'X_timeseries.pkl', 'wb') as fout:
            pickle.dump(X, fout)
        with open(cfg.ready_dir + 'Y_timeseries.pkl', 'wb') as fout:
            pickle.dump(Y, fout)
    else:
        print("------------ Loading Time Series data ------------")
        with open(cfg.ready_dir + 'X_timeseries.pkl', 'rb') as fout:
            X = pickle.load(fout)
        with open(cfg.ready_dir + 'Y_timeseries.pkl', 'rb') as fout:
            Y = pickle.load(fout)
    X = np.vstack(X)
    Y = np.expand_dims(np.vstack(Y), axis=-1)
    X_df.drop('year', axis='columns', inplace=True)
    for fold_n in range(cfg.MLP_folds):
        print('RMLP:', fold_n)
        X_train, X_valid, y_train, y_valid = train_test_split(X, Y, test_size=1 / cfg.MLP_folds)
        X_train = get_keras_data_RMLP(X_train, X_df.columns) # TODO delete if we return to prior version without embedding
        X_valid = get_keras_data_RMLP(X_valid, X_df.columns)
        keras_rmlp = RMLP(X_df.columns)
        keras_rmlp.summary()
        trained_rmlp = train_model(keras_model=keras_rmlp, X_t=X_train, y_train=y_train,
                                   batch_size=cfg.batch_size, epochs=epochs, patience=20,
                                   X_v=X_valid, y_valid=y_valid, fold=fold_n, model_dir=cfg.RMLP_dir,
                                   schedule=cfg.learning_rate_scheduler_RMLP)
        models.append(trained_rmlp)
        print('*' * 50)

if __name__ == '__main__':
    train_X_total = pd.read_pickle(path=cfg.ready_dir + 'train_X.pkl')
    train_Y_total = pd.read_pickle(path=cfg.ready_dir + 'train_y.pkl')
    train_X_total = preprocess_data_for_RMLP(X_train=train_X_total, RMLP=True)
    train_models_using_RMLP(train_X_total, train_Y_total, first=False)
    # plotting_train_RMLP_prediction(train_X_total, train_Y_total)
    test_X = pd.read_pickle(path=cfg.ready_dir + 'test_X.pkl')
    test_X = preprocess_data_for_RMLP(test_X, RMLP=True, test_signal=True)
    evaluate_RMLP(test_X, first=False)


import os

import urllib

from google.appengine.api import users
from google.appengine.ext import ndb          # To use the data modeling API, imports the google.appengine.ext.ndb module

import jinja2
import webapp2

template_dir = os.path.join(os.path.dirname(__file__),'template')
jinja_env = jinja2.Environment(loader = jinja2.FileSystemLoader(template_dir), autoescape = True)

DEFAULT_GUESTBOOK_NAME = 'default_guestbook'
COMMENTSNUMBER = 10
ERROR_MSG = ''

# We set a parent key on the 'Greetings' to ensure that they are all
# in the same entity group. Queries across the single entity group
# will be consistent.  However, the write rate should be limited to
# ~1/second.

def guestbook_key(guestbook_name=DEFAULT_GUESTBOOK_NAME):
    """Constructs a Datastore key for a Guestbook entity.

    We use guestbook_name as the key.
    """
    return ndb.Key('Guestbook', guestbook_name)

class Author(ndb.Model):
    """Sub model for representing an author."""
    identity = ndb.StringProperty(indexed=False)
    email = ndb.StringProperty(indexed=False)

class Greeting(ndb.Model):
    """A main model for representing an individual Guestbook entry."""
    author = ndb.StructuredProperty(Author)
    content = ndb.StringProperty(indexed=False)
    date = ndb.DateTimeProperty(auto_now_add=True)

class MainPage(webapp2.RequestHandler):
    def get(self):

        guestbook_name = self.request.get('guestbook_name',
                                          DEFAULT_GUESTBOOK_NAME)
        error_msg = self.request.get('error_msg',
                                          ERROR_MSG)

        # Ancestor Queries, as shown here, are strongly consistent
        # with the High Replication Datastore. Queries that span
        # entity groups are eventually consistent. If we omitted the
        # ancestor from this query there would be a slight chance that
        # Greeting that had just been written would not show up in a
        # query.
        greetings_query = Greeting.query(
            ancestor=guestbook_key(guestbook_name)).order(-Greeting.date)
        greetings = greetings_query.fetch(COMMENTSNUMBER)

        user = users.get_current_user()

        if user:
            url = users.create_logout_url(self.request.uri)
            url_linktext = 'Logout'
        else:
            url = users.create_login_url(self.request.uri)
            url_linktext = 'Login'

        # Write the submission form and the footer of the page
        sign_query_params = urllib.urlencode({'guestbook_name':
                                              guestbook_name})
        template_values = {
            'user': user,
            'greetings': greetings,
            'guestbook_name': urllib.quote_plus(guestbook_name),
            'url': url,
            'url_linktext': url_linktext,
            'error_msg':error_msg,

        }

        template = jinja_env.get_template('stage4.html')
        self.response.write(template.render(template_values))

class Guestbook(webapp2.RequestHandler):
    def post(self):
        # We set the same parent key on the 'Greeting' to ensure each
        # Greeting is in the same entity group. Queries across the
        # single entity group will be consistent. However, the write
        # rate to a single entity group should be limited to
        # ~1/second.
        guestbook_name = self.request.get('guestbook_name',
                                          DEFAULT_GUESTBOOK_NAME)
        error_msg = self.request.get('error_msg',
                                          ERROR_MSG)
        greeting = Greeting(parent=guestbook_key(guestbook_name))

        if users.get_current_user():
            greeting.author = Author(
                    identity=users.get_current_user().user_id(),
                    email=users.get_current_user().email())

        greeting.content = self.request.get('content')

        if greeting.content  and not greeting.content.isspace():
            greeting.put()
            error_msg = ""
        else:
            error_msg = "please add a validated message"

        query_params = {'guestbook_name': guestbook_name, 'error_msg':error_msg}
        self.redirect('/?' + urllib.urlencode(query_params))

app = webapp2.WSGIApplication([
    ('/', MainPage),
    ('/sign', Guestbook),
], debug=True)


"""
####### Customized implementation of the Kalman Filter ###########
"""

import numpy as np
from scipy.optimize import fmin
import copy

class KalmanFilter(object):
    """
    Implements a Kalman Filter
    """
    def __init__(self,
             A = None, B = None, C = None,          # System Dynamics
             SigmaXXpred0 = None, Xpred0 = None,    # Initial prediction
             SystemCovNoise = None, MeasurCovNoise = None):  # System and Measuring noise
        """
        # System of equations A
        #   Xt = A*Xt-1 + B*Ut + SystemNoise
        #   Yt = Cx_k  + MeasurementNoise
        # In the normal formulation Xt is a column vector !!
        """
        self.A,self.B,self.C = A,B,C
        self.SigmaXXpred0, self.Xpred0 = SigmaXXpred0, Xpred0
        self.SystemCovNoise, self.MeasurCovNoise = SystemCovNoise, MeasurCovNoise

    def init_KF(self, y):
        ####################### Initialize Kalman Filter ##########################
        ## Time to choose our priors of prediction !!
        ## Important:
        #      - Variables with "hat" are estimated variables (V(t|t))
        #      - Variables with "pred" are predicted variables (V(t+1|t))
        # In the begining we have to define Xpred(1|0).
        # Observed samples start at time t = 1. t = 0 is our prior thingy.

        ## We just initialize the parameters of the kalman it with a good guess.
        # In this case we just use the initial obervation of Yo as mean of the prior. X0hat = Y0
        Nd,Ns = y.shape
        Xpred0 = y[:,[0]] # Initilization Xpred for X1 given Y0

        # Initilize the uncertainty of the first samples high enough if we are not sure
        # or low enough if we are sure of out initial Xhat
        # As more samples arise, this influence of this initial decision will be decrease.
        # We choose a small uncertainty of the initializatoin
        varpred0 = 20
        SigmaXXpred0  = np.eye(Nd) * varpred0
        return Xpred0,SigmaXXpred0

    def init_fit_param(self, X, y,A, B, C,
             SigmaXXpred0, Xpred0,SystemCovNoise, MeasurCovNoise):
        # This funciton initializices properly the parameters when callinga fit.
        ## In this part we just select between the ones given and the inner ones
        if (type(A) == type(None)):
            A = self.A
        if (type(B) == type(None)):
            B = self.B
        if (type(C) == type(None)):
            C = self.C

        if (type(SigmaXXpred0) == type(None)):
            SigmaXXpred0 = self.SigmaXXpred0
        if (type(Xpred0) == type(None)):
            Xpred0 = self.Xpred0

        if (type(SystemCovNoise) == type(None)):
            SystemCovNoise = self.SystemCovNoise
        if (type(MeasurCovNoise) == type(None)):
            MeasurCovNoise = self.MeasurCovNoise

        # If initial conditions are not given, we use
        #   - The initial value as the prediction
        #   - A small variance
        if (type(Xpred0) == type(None)):
            Xpred0, SigmaXXpred0 = self.init_KF(y)

        return A,B,C, Xpred0, SigmaXXpred0, SystemCovNoise, MeasurCovNoise

    def fit (self, X, y,                        # Data
             A = None, B = None, C = None,      # System Dynamics
             SigmaXXpred0 = None, Xpred0 = None, # Initial prediction
             SystemCovNoise = None, MeasurCovNoise = None):  # System and Measuring noise
        """
        This function runs the Kalman filter once over the data given
        and returns the data
        """
        self.X = X
        self.Y = y
        # Now the magic starts :)
        Ns,Nd = y.shape
        y = y.T # So we need to transpose our normal X[Nsam,Ndim]

        A,B,C, Xpred0, SigmaXXpred0, SystemCovNoise, MeasurCovNoise = self.init_fit_param( X, y,A, B, C,
             SigmaXXpred0, Xpred0,SystemCovNoise, MeasurCovNoise)

        ######### Initialize variables just to keep intermediate results ##########
        XhatList = []               # Store here the estimations of X
        SigmaXXhatList = []        # Store here the estimated Sigma of X
        XpredList = []              # Store here the predictions of X
        SigmaXXpredList = []        # Store here the predicted Sigma of X

        XpredList.append(Xpred0)           # Store here the predictions of X
        SigmaXXpredList.append(SigmaXXpred0)       # Store here the predicted Sigma of X

        ######## Initialize the first step data  ########
        Xpred = Xpred0
        SigmaXXpred = SigmaXXpred0
        ## Calculate covariance matrix between observarionts and latent variables
        SigmaXYpred  = SigmaXXpred.dot(C.T)
        # Maybe same as the measurement noise would be appropiate.
        SigmaYYpred = C.dot(SigmaXYpred) + MeasurCovNoise

        ################## RECONSTRUCTION OF THE STATE and ONE prediction #################
        for n in range (0,Ns):
          #############   ESTIMATING STEP  ###################
          # Estimation of theparameters ("hat" variables V(t|t))
          # Calculated from the predicted values and the new sample
          K = SigmaXYpred.dot(np.linalg.inv(SigmaYYpred))  # Kalman Gain
          XpredError = y[:,[n]]- Xpred
          Xhat = Xpred + K.dot(C.dot(XpredError))
          SigmaXXhat = SigmaXXpred - K.dot(SigmaYYpred.dot(K.T))
          #############   PREDICTION STEP  ###################
          ## Predict the variables for the next instance  V(t+1|t)
          Xpred = A.dot(Xhat)  # We use the correlation bit
          SigmaXXpred = A.dot(SigmaXXhat.dot(A.T)) + SystemCovNoise
          SigmaXYpred = SigmaXXpred.dot(C.T)
          SigmaYYpred = C.dot(SigmaXYpred) + MeasurCovNoise
          ################# Storing data  ################
          XhatList.append(Xhat)
          SigmaXXhatList.append(SigmaXXhat)
          XpredList.append(Xpred)
          SigmaXXpredList.append(SigmaXXpred)

        ############ Restructure Data in normal np format #######
        XhatList = np.concatenate(XhatList, axis = 1)
        XpredList = np.concatenate(XpredList, axis = 1)
        Yhat =  C.dot(XhatList)
        Ypred = C.dot(XpredList)

        Yhat = Yhat.T
        Ypred = Ypred.T

        # Set data as local variables
        self.SigmaXXhatList = SigmaXXhatList
        self.SigmaXXpredList = SigmaXXpredList
        self.Yhat = Yhat
        self.Ypred = Ypred

        return copy.deepcopy([Yhat,SigmaXXhatList,Ypred, SigmaXXpredList])

    def predict(self, Xpredtest0 = None, SigmaXXpredtestList0 = None,
                Ntst = 10):
        """
        This function predicts the future of the Kalman from the last estimation.
           - If param are given, those are used.
           - If not, the local variables are used.
           - If the local variables do not exist, then we should have called fit first
        """

        if (type(Xpredtest0) == type(None)):
            Xpredtest = self.Ypred[[-1],:].T
            SigmaXXpredtest = self.SigmaXXpredList[-1]
        else:
            Xpredtest= Xpredtest0
            SigmaXXpredtest = SigmaXXpredtestList0

        ## Variables to store the results
        XpredtestList = []
        SigmaXXpredtestList = []

        # The first prediction is the one calculated last in the previous loop
        XpredtestList.append( Xpredtest)
        SigmaXXpredtestList.append(SigmaXXpredtest)

        # We calculate the rest
        A,C,SystemCovNoise = self.A, self.C, self.SystemCovNoise

#        print SigmaXXpredtest.shape
        for n in range(Ntst):
          # Perform future predictions
          Xpredtest = A.dot(Xpredtest)
          SigmaXXpredtest = A.dot(SigmaXXpredtest.dot(A.T)) + SystemCovNoise

          XpredtestList.append(Xpredtest)
          SigmaXXpredtestList.append(SigmaXXpredtest)

        ##################  Transform data and preprocess  ################################
        XpredtestList = np.concatenate(XpredtestList, axis = 1)
        Ypredtest = C.dot(XpredtestList)
        ## Now we transpose things back to our normal Nsamxdim
        Ypredtest = Ypredtest.T
        # Set data as local variables
        self.Ypredtest = Ypredtest
        self.SigmaXXpredtestList = SigmaXXpredtestList

        return copy.deepcopy([Ypredtest,SigmaXXpredtestList])

    def negative_ll(self, params ,*args):
        """ This funcion gives us the negative_ll of the data given the model"""

        X,y = args
        varPrice, varPriceDiff, varPriceNoise = params
        SystemCovNoise, MeasurCovNoise = self.build_CovMatrix(y, varPrice, varPriceDiff, varPriceNoise)

#        print SystemCovNoise
        Yhat,SigmaXXhatList,Ypred, SigmaXXpredList = self.fit(X,y, SystemCovNoise = SystemCovNoise, MeasurCovNoise = MeasurCovNoise)

        # Compute the YY variance
        C = self.C
        SigmaYYpredList = []
        for i in range(len(SigmaXXpredList)):
            SigmaXXpred = SigmaXXpredList[i]
            SigmaXYpred = SigmaXXpred.dot(C.T)
            SigmaYYpred = C.dot(SigmaXYpred) + MeasurCovNoise
            SigmaYYpredList.append(SigmaYYpred)

        nll = self.negative_ll_func(y, Ypred[:-1,:], SigmaYYpredList)

        return nll

    def negative_ll_func(self, y, ypred, SigmaYYpredList):
    # Returns the log-likelihood of the likelihood of the prediction given
    # the model and the current state P(Ypred | Ypred)
        nll = 0
        Ns,Ndim = y.shape

        yerr = ypred - y  # Difference at each point, between the predicted and the real.
        for i in range (Ns):
            yerr_i = yerr[[i],:]
            nll += np.log(np.linalg.det(SigmaYYpredList[i]))
            nll += (yerr_i).dot(np.linalg.inv(SigmaYYpredList[i])).dot(yerr_i.T)

        nll += Ns*np.log(np.pi * 2)

        nll = nll/2
        return nll

    def optimize_parameters(self, varPrice, varPriceDiff, varPriceNoise):

        """ This function should optimize the parameters """
        # We give as intial guess the desired values
        xopt = fmin(func= self.negative_ll,
                    x0 = np.array([varPrice, varPriceDiff, varPriceNoise]),
                    args=(self.X,self.Y))

        return xopt

    def build_CovMatrix(self, y, varPrice, varPriceDiff,varPriceNoise):
        """ Returns the proper Covariance matrix from a set of parameters """
        Ns,Nd = y.shape
        SystemCovNoise = np.eye(Nd) *varPrice
        SystemCovNoise[1,1] = varPriceDiff
        MeasurCovNoise = np.eye(Nd) *varPriceNoise

        return SystemCovNoise, MeasurCovNoise


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('romani', '0072_auto_20170214_1219'),
    ]

    operations = [
        migrations.AddField(
            model_name='producte',
            name='frequencies',
            field=models.ManyToManyField(to='romani.Frequencia', null=True, blank=True),
        ),
    ]


from django.db import models
import datetime
from django.contrib.auth.models import User

# Create your models here.

class LibUser(User):
    PROVINCE_CHOICES = (
        ('AB','Alberta'),
        ('MB', 'Manitoba'),
        ('ON', 'Ontario'),
        ('QC', 'Quebec'),

    )
    address = models.CharField(max_length=100, null=True, blank=True)
    city = models.CharField(max_length=20, default='Windsor')
    province = models.CharField(max_length=2, choices=PROVINCE_CHOICES, default='ON')
    phone = models.IntegerField(null=True)
    postalcode = models.CharField(null=True,blank=True,max_length=7)
    upload = models.FileField(null=True,blank=True,upload_to='uploads/%Y/%m/%d/')

class Libitem(models.Model):
    TYPE_CHOICES = (
        ('Book', 'Book'),
        ('DVD','DVD'),
        ('Other', 'Other'),
    )
    title = models.CharField(max_length=100)
    itemtype = models.CharField(max_length=6, choices=TYPE_CHOICES, default='Book')
    checked_out=models.BooleanField(default=False)
    user=models.ForeignKey(LibUser, default=None, null=True, blank=True)
    duedate=models.DateField(default=None, null=True, blank=True)
    last_chkout = models.DateField(default=None, null=True, blank=True)
    date_acquired = models.DateField(default=datetime.date.today())
    pubyr = models.IntegerField()
    num_chkout = models.IntegerField(blank=False,default=0)

    def overdue(self):
        if self.checked_out == True and self.duedate < datetime.date.today():
            return 'Yes'
    overdue.short_description = 'Overdue'

class Book(Libitem):
    CATEGORY_CHOICES = (
        (1, 'Fiction'),
        (2, 'Biography'),
        (3, 'Self Help'),
        (4, 'Education'),
        (5, 'Children'),
        (6, 'Teen'),
        (7, 'Other'),
    )
    author = models.CharField(max_length=100)
    category = models.IntegerField(choices=CATEGORY_CHOICES, default=1)

    def __str__(self):
        return self.title + ' by ' + self.author

class Dvd(Libitem):
    maker = models.CharField(max_length=100)
    duration = models.IntegerField()
    rating_choices = (
        (1,'G'),
        (2,'PG'),
        (3,'PG-13'),
        (4,'14A'),
        (5,'R'),
        (6,'NR'),
    )
    rating = models.IntegerField(choices=rating_choices,default=1)

    def __str__(self):
        return self.title + ' by ' + self.maker

class Suggestion(models.Model):

    TYPE_CHOICES = (
        (1, 'Book'),
        (2,'DVD'),
        (3, 'Other'),
    )
    title = models.CharField(max_length=100)
    pubyr = models.IntegerField(null=True, blank=True)
    type = models.IntegerField(default=1, choices=TYPE_CHOICES)
    cost = models.IntegerField()
    num_interested = models.IntegerField()
    comments = models.TextField(blank=True)

    def __str__(self):
        return self.title

class UserProfile(models.Model):
    user = models.OneToOneField(User)
    picture = models.ImageField(upload_to='profile_images',blank=True)

    def __str__(self):
        return self.user.username


from fastapi import FastAPI
import uvicorn
from routs import facts_about,mails

app = FastAPI()

app.include_router(facts_about.router)
app.include_router(mails.router)

if __name__ == "__main__":
    uvicorn.run(
        "main:app",
        host="0.0.0.0",
        reload=True,
        port=8080
    )


import os
import shutil
import openpyxl

def searchZone(name):
    wb = openpyxl.load_workbook("/Users/Wendy/学生地区.xlsx")
    sheet = wb["地区表"]
    for row in sheet.rows:
        if row[0].value == name:
            return row[1].value

path = "/Users/Wendy/学生资料"
allItems = os.listdir(path)
for item in allItems:
    filePath = os.path.join(path,item)
    name = os.path.splitext(item)[0]
    zone = searchZone(name)
    targetPath = os.path.join(path,zone)
    if not os.path.exists(targetPath):
        os.mkdir(targetPath)
    shutil.move(filePath,targetPath)

根据学生地区移动文件.py


import os
import numpy as np
import params_priors as p
import matplotlib.pyplot as plt
import time
import matplotlib.animation as animation
#import priors_tabular as PR

def Qlearn_multirun_tab():
	retlog=[]
	log_leftcountperc=[]
	QAloglog=[]
	for i in range(p.Nruns):
		print("Run no:",i)
		QA,QB,ret,QAlog=main_Qlearning_tab()
		if i==0:
			retlog=ret
			QAloglog=QAlog
			#log_leftcountperc=leftcountperc
		else:
			retlog=np.vstack((retlog,ret))
			QAloglog=np.vstack((QAloglog,QAlog))
			#log_leftcountperc=np.vstack((log_leftcountperc,leftcountperc))
		if (i+1)/p.Nruns==0.25:
			print('25% runs complete')
		elif (i+1)/p.Nruns==0.5:
			print('50% runs complete')
		elif (i+1)/p.Nruns==0.75:
			print('75% runs complete')
		elif (i+1)==p.Nruns:
			print('100% runs complete')
	meanreturns=(np.mean(retlog,axis=0))
	#meanleft=(np.mean(log_leftcountperc,axis=0))
	plt.plot(meanreturns)
	plt.show()
	return QA,QB,retlog,QAloglog

def main_Qlearning_tab():
	QA=np.zeros((1,p.NQ,p.A_asize))#np.random.uniform(low=-0, high=0.01, size=(1,))*np.ones((1,p.NQ,p.A_asize))
	QB=np.zeros((1,p.NQ,p.B_asize))#np.random.uniform(low=-0, high=0.01, size=(1,))*np.ones((1,p.NQ,p.B_asize))#np.zeros((1,p.NQ,p.B_asize))
	returns=[]
	QAlog =[]
	for i in range(p.episodes):
		if (i+1)/p.episodes==0.25:
			print('25% episodes done')
		elif (i+1)/p.episodes==0.5:
			print('50% episodes done')
		elif (i+1)/p.episodes==0.75:
			print('75% episodes done')
		elif (i+1)/p.episodes==1:
			print('100% episodes done')
			#print(fr)
		QA,QB,ret,QAval=Qtabular(QA,QB,i)
		if i%1==0:
			returns.append(ret)
			QAlog.append(QAval)
	return QA,QB,returns,QAlog

def transition(state,act):
	if state==p.stateA:
		if act==0:
			new_state=p.stateB
		elif act==1:
			new_state=p.state_term_neur
	if state==p.stateB:
		new_state=p.state_term_rew
	return new_state

def maxQ_tab(QA,QB,state):
	Qlist=[]
	if state==p.stateA:
		asize=p.A_asize
		Q=QA[0]
		for i in range(asize):
			Qlist.append(Q[i])
		tab_maxQ=np.max(Qlist)
	elif state==p.stateB:
		asize=p.B_asize
		Q=QB[0]
		for i in range(asize):
			Qlist.append(Q[i])
		tab_maxQ=np.max(Qlist)
	if state==p.stateA or state==p.stateB:
		maxind=[]
		for j in range(len(Qlist)):
			if tab_maxQ==Qlist[j]:
				maxind.append(j)
		if len(maxind)>1:
			optact=maxind[np.random.randint(len(maxind))]
		else:
			optact=maxind[0]
		tab_minQ=np.min(Qlist)
	else:
		tab_minQ=0
		tab_maxQ=0
		optact=0
	return tab_maxQ,tab_minQ, optact

def env_step(state,action):
	if state==p.state_term_rew:
		reward=p.meanreward+np.random.uniform(-p.uncertain,p.uncertain,1)
	else:
		reward=0
	return reward

def epsilon_greedy(state,eps_live,QA,QB):
	if state==p.stateA:
		asize=p.A_asize
		if eps_live>np.random.sample():
			a=np.random.randint(asize)
		else:
			Qmax,Qmin,a=maxQ_tab(QA,QB,state)
	elif state==p.stateB:
		asize=p.B_asize
		if eps_live>np.random.sample():
			a=np.random.randint(asize)
		else:
			Qmax,Qmin,a=maxQ_tab(QA,QB,state)
	return a

def minQN(QA,QB,state):
	min_QN=[]
	if state==p.stateA:
		for j in range(p.A_asize):
			Qall=[]
			for i in range(p.NQ):
				Qall.append(QA[0][i][j])
			min_QN.append(np.min(Qall))
			#min_QN_B=QB[0][arg]
	elif state==p.stateB:
		for j in range(p.B_asize):
			Qall=[]
			for i in range(p.NQ):
				Qall.append(QB[0][i][j])
			min_QN.append(np.min(Qall))
			#min_QN_B=QB[0][arg]
	return min_QN

def minQNmod(QA,QB,state):
	min_QNA=[]
	min_QNB=[]
	for j in range(p.A_asize):
		Qall=[]
		for i in range(p.NQ):
			Qall.append(QA[0][i][j])
		min_QNA.append(np.min(Qall))
		#min_QN_B=QB[0][arg]

	for j in range(p.B_asize):
		Qall=[]
		for i in range(p.NQ):
			Qall.append(QB[0][i][j])
		min_QNB.append(np.min(Qall))
		#min_QN_B=QB[0][arg]
	return min_QNA, min_QNB

def epsilon_greedy_min(state,eps_live,QA,QB):
	if state==p.stateA:
		min_QNA,min_QNB=minQNmod(QA,QB,state)
		asize=p.A_asize
		if eps_live>np.random.sample():
			a=np.random.randint(asize)
		else:
			Qmax,a=maxQ_tab_newnew(min_QNA,min_QNB,state)

	elif state==p.stateB:
		min_QNA,min_QNB=minQNmod(QA,QB,state)
		bsize=p.B_asize
		if eps_live>np.random.sample():
			a=np.random.randint(bsize)
		else:
			Qmax,a=maxQ_tab_newnew(min_QNA,min_QNB,state)
	return a

def epsilon_greedy(state,eps_live,QA,QB):
	if state==p.stateA:
		asize=p.A_asize
		if eps_live>np.random.sample():
			a=np.random.randint(asize)
		else:
			Qmax,Qmin,a=maxQ_tab(QA,QB,state)
	elif state==p.stateB:
		asize=p.B_asize
		if eps_live>np.random.sample():
			a=np.random.randint(asize)
		else:
			Qmax,Qmin,a=maxQ_tab(QA,QB,state)
	return a

def maxQ_tab_new(Qcurr,state):

	Qlist=[]
	if state==p.stateA or state==p.stateB:
		if state==p.stateA:
			asize=p.A_asize
		elif state==p.stateB:
			asize=p.B_asize
		for i in range(asize):
			Qlist.append(Qcurr[i])
		Qmaxnext=np.max(Qlist)
		optact=np.argmax(Qlist)
	else:
		Qmaxnext=0
		optact=0
	return Qmaxnext,optact

def maxQ_tab_newnew(QcurrA,QcurrB,state):

	Qlist=[]
	if state==p.stateA:
		asize=p.A_asize
		for i in range(asize):
			Qlist.append(QcurrA[i])
		Qmaxnext=np.max(Qlist)
		maxinds=[]
		for j in range(len(Qlist)):
			if Qlist[j]==Qmaxnext:
				maxinds.append(j)
		if len(maxinds)>1:
			optact=maxinds[np.random.randint(len(maxinds))]
		else:
			optact=maxinds[0]
	elif state==p.stateB:
		asize=p.B_asize
		for i in range(asize):
			Qlist.append(QcurrB[i])
		Qmaxnext=np.max(Qlist)
		maxinds=[]
		for j in range(len(Qlist)):
			if Qlist[j]==Qmaxnext:
				maxinds.append(j)
		if len(maxinds)>1:
			optact=maxinds[np.random.randint(len(maxinds))]
		else:
			optact=maxinds[0]
	else:
		Qmaxnext=0
		optact=0
	return Qmaxnext,optact

def findmeanval(QA,QB):
	meanval=(np.sum(QA[0][0:p.A_asize])+np.sum(QB[0][0:p.B_asize]))/(p.A_asize+p.B_asize)
	return meanval

def Qtabular(QA,QB,episode_no):
	initial_state=p.stateA
	state=initial_state
	count=0
	breakflag=0
	#eps_live=1-(p.epsilon_decay*episode_no)
	eps_live=0.1
	doneflag=0
	ret=0
	leftcount=0
	Vmax=0
	Vmin=0
	count=0
	while doneflag==0:
		if state==p.state_term_neur or state==p.state_term_rew:
			break
		a=epsilon_greedy_min(state,eps_live,QA,QB)
		next_state=transition(state,a)
		R=env_step(next_state,a)
		ret=ret+R
		minQnextA,minQnextB=minQNmod(QA,QB,next_state)
		Qmaxnext,optact=maxQ_tab_newnew(minQnextA,minQnextB,next_state)
		rng=np.random.default_rng()
		numbers = rng.choice(p.NQ, size=1, replace=False)
		for i in range(len(numbers)):
			QsubA=QA[0][numbers[i]]
			QsubB=QB[0][numbers[i]]
			if state==p.stateA:
				count=count+1
				#QA[0][a]=QA[0][a]+p.alpha*(R+(p.gamma*Qminnext)-QA[0][a])
				QsubA[a]=QsubA[a]+p.alpha*(R+(p.gamma*(Qmaxnext))-QsubA[a])
				if a==0:
					leftcount=leftcount+1
			elif state==p.stateB:
				#QB[0][a]=QB[0][a]+p.alpha*(R+(p.gamma*Qminnext)-QB[0][a])
				QsubB[a]=QsubB[a]+p.alpha*(R+(p.gamma*Qmaxnext)-QsubB[a])
			QA[0][numbers[i]]=QsubA
			QB[0][numbers[i]]=QsubB
		percleft=QA[0][0][0]-QA[0][0][1]
		if percleft<=0:
			percleft=0
		if state==p.state_term_neur or state==p.state_term_rew:
			doneflag=1
			break
		state=next_state
		#count=count+1
	return QA,QB,ret,leftcount/count

#######################################
if __name__=="__main__":
	QA,QB,retlog,QAloglog=Qlearn_multirun_tab()
	np.savez(str(p.NQ)+"maxmin_"+str(p.Nruns)+"runs_R_"+str(p.meanreward)+"uncertain_"+str(p.uncertain)+".npy.npz",QA,QB,retlog,QAloglog)


import cv2
import processing
import table
import balls
import numpy as np

def get_balls(LL, UL, LR, UR):
    image_path = "./images/IMG_2737.png"
    color = cv2.imread(image_path, 1)
    masked_img = processing.mask_table(color)
    corners = [(c[0], c[1]) for c in table.get_corners(masked_img=masked_img)]
    blurred_img = processing.blur_img(masked_img, iters=9)
    sch = processing.scharr(blurred_img)
    circles = balls.hough_circles(color, sch, bounds=corners)
    bls = balls.label_balls(color, circles)
    # the src_corners are in order of TL, TR, BL, BR
    src_corners = np.float32(np.array(corners))
    # the dst_corners are the coordinates according to our physics engine
    dst_corners = np.float32(np.array([(70, 510), (70, 70), (986, 510), (986, 70)]))
    M = cv2.getPerspectiveTransform(src_corners, dst_corners)
    src_balls = np.array([ball.loc for ball in bls])
    src_balls = np.array([src_balls], np.float32)
    dest_balls = cv2.perspectiveTransform(src_balls, M)
    warped_img = cv2.warpPerspective(color, M, (1100, 600), flags=cv2.INTER_LINEAR)
    cv2.imwrite("warped_img.png", warped_img)
    for i in range(len(bls)):
        bls[i].loc = dest_balls[0, i]

    # THESE BALLS HAVE ALREADY BEEN MAPPED TO THE PHYSICS COORDINATE SPACE
    return bls


import tensorflow as tf
from datetime import datetime
import os
import sys
import data_sampling.data_args as data_args
from nets.two_stream import TwoStreamNet
import constants as const
import configuration as file_const
from data_sampling.tuple_loader import TupleLoader
import numpy as np
import utils
import data_sampling.honda_tuple_generator as honda_gen
import data_sampling.preprocess as preprocess
from sklearn.metrics import confusion_matrix
import itertools

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

def gen_feed_dict(model,data_generator,sess,data_generator_reset):
    #print('Going to get new feeddict')
    try:
        data  = sess.run(data_generator)
    except:
        print('Need to reset data generator')
        sess.run(data_generator_reset)
        data = sess.run(data_generator)
    #print('Pre processing data')
    words , context,lbl= preprocess.data_prepare(data);
    #print('Pre processing done')
    # print('words.shape ',words.shape)
    # print('context.shape ', context.shape)
    # print('lbl.shape ', lbl.shape)
    feed_dict = {model.input_words: words, model.input_context: context, model.supervised_labels: lbl}
    #print('feeddict constructed')
    return feed_dict;

def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

if __name__ == '__main__':

    save_model_dir = file_const.model_save_path;

    args = dict()
    args[data_args.gen_nearby_frame] = False;
    args[data_args.data_augmentation_enabled] = False

    with open(file_const.honda_session_path+'/train_session.txt', 'r') as fin:
        train_sessions = fin.read().strip().split('\n')
    with open(file_const.honda_session_path+'/val_session.txt', 'r') as fin:
        val_sessions = fin.read().strip().split('\n')

    train_data = honda_gen.tuple_generator(supervised=True, sessions=train_sessions,batch_size=const.batch_size)
    #    train_data = tuple_generator(supervised=False, sessions=train_sessions)
    train_iterator = tf.data.Iterator.from_structure(train_data.output_types,
                                                     train_data.output_shapes)
    next_train = train_iterator.get_next()
    train_init_op = train_iterator.make_initializer(train_data)

    val_data = honda_gen.tuple_generator(supervised=True, sessions=val_sessions, shuffled=False,batch_size=const.batch_size)
    #    val_data = tuple_generator(supervised=False, sessions=val_sessions, shuffled=False)
    val_iterator = tf.data.Iterator.from_structure(val_data.output_types,
                                                   val_data.output_shapes)
    next_val = val_iterator.get_next()
    val_init_op = val_iterator.make_initializer(val_data)

    sess = tf.InteractiveSession()
    tf.global_variables_initializer().run()
    sess.run(train_init_op)

    load_alex_weights = True;

    img2vec_model = TwoStreamNet(supervised=True,load_alex_weights=load_alex_weights,train_spatial_tower = False,train_motion_tower = True)

    #feed_dict = gen_feed_dict(img2vec_model, next_train, sess,train_init_op);

    model_loss = img2vec_model.supervised_loss
    model_accuracy = img2vec_model.supervised_accuracy

    optimizer = tf.train.AdamOptimizer(const.learning_rate, beta1=0.5)
    grads = optimizer.compute_gradients(model_loss)

    for i, (g, v) in enumerate(grads):
        if g is not None:
            grads[i] = (tf.clip_by_norm(g, 5), v)

    train_op = optimizer.apply_gradients(grads)

    for v in tf.trainable_variables():
        print(v.name , '\t',v.shape)

    sess = tf.InteractiveSession()
    now = datetime.now()
    if(file_const.tensorbaord_file == None):
        tb_path = file_const.tensorbaord_dir + now.strftime("%Y%m%d-%H%M%S")
    else:
        tb_path = file_const.tensorbaord_dir + file_const.tensorbaord_file

    print(tb_path)
    if (os.path.exists(tb_path)):
        latest_filepath = utils.get_latest_file(tb_path)
        print(latest_filepath)
        tb_iter = tf.train.summary_iterator(latest_filepath)
        for e in tb_iter:
            last_step = e.step;
        print('Continue on previous TB file ', tb_path, ' with starting step', last_step);
    else:
        print('New TB file *********** ', tb_path);
        last_step = 0;

    train_writer = tf.summary.FileWriter(tb_path, sess.graph)
    tf.global_variables_initializer().run()
    saver = tf.train.Saver()  # saves variables learned during training

    #sess.run(img2vec_model.assign_operations)
    #img2vec_model.print_means(sess);
    ckpt_file = os.path.join(save_model_dir, file_const.model_save_name)
    cnf_file = os.path.join(save_model_dir, 'cnf_matrix.png');
    print('Model Path ',ckpt_file )
    if (os.path.exists(save_model_dir) and len(os.listdir(save_model_dir)) > 1):
        try:
            # Try to restore everything if possible
            saver.restore(sess, ckpt_file)
            print('Model Loaded Normally');
        except:
            ## If not, load as much as possible
            img2vec_model.load_pretrained(sess, ckpt_file);
            print('Pretrained Weights loaded, while some layers are randomized')

    elif load_alex_weights:
        print('Loading img2vec_model.assign_operations:',len(img2vec_model.assign_operations));
        sess.run(img2vec_model.assign_operations);

    train_loss = tf.summary.scalar('Train Loss', model_loss)
    val_loss = tf.summary.scalar('Val Loss', model_loss)
    model_acc_op = tf.summary.scalar('Val Accuracy', model_accuracy)

    sess.run(train_init_op)
    sess.run(val_init_op)
    confusion_matrix_gt_labels = np.array([]);
    confusion_matrix_predictions = np.array([]);
    for step in range(last_step,const.train_iters):

        feed_dict = gen_feed_dict(img2vec_model, next_train,sess,train_init_op);
        model_loss_value,accuracy_value, _ = sess.run([model_loss,model_accuracy,train_op], feed_dict)

        if(step % const.logging_threshold == 0):
            print('i= ', step, ' Loss= ', model_loss_value, ', Acc= %2f' % accuracy_value);
            if(step != 0):
                run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
                run_metadata = tf.RunMetadata()

                feed_dict = gen_feed_dict(img2vec_model, next_train,sess,train_init_op);
                train_loss_op,_= sess.run([train_loss,train_op],feed_dict=feed_dict)

                feed_dict = gen_feed_dict(img2vec_model, next_val,sess,val_init_op);
                val_loss_op,accuracy_op= sess.run([val_loss,model_acc_op], feed_dict=feed_dict)

                predictions = sess.run(img2vec_model.class_prediction, feed_dict)
                gt_labels = np.argmax(feed_dict[img2vec_model.supervised_labels], axis=1)

                confusion_matrix_gt_labels = np.append(confusion_matrix_gt_labels, gt_labels)
                confusion_matrix_predictions = np.append(confusion_matrix_predictions, predictions)

                train_writer.add_run_metadata(run_metadata, 'step%03d' % step)

                train_writer.add_summary(train_loss_op, step)
                train_writer.add_summary(val_loss_op, step)

                train_writer.add_summary(accuracy_op, step)
                train_writer.flush()

                if(step % 100 == 0):

                    saver.save(sess, ckpt_file)

                    confusion_matrix_labels = ["Background", "Intersection", "L turn", 'R turn',
                                               'L ln change', 'R ln change', 'Crosswalk',
                                               'U-turn', 'L ln branch', 'R ln branch', 'Merge']

                    cnf_matrix = confusion_matrix(confusion_matrix_gt_labels, confusion_matrix_predictions,
                                                  labels=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
                    #print(confusion_matrix)

                    # Plot non-normalized confusion matrix
                    plt.figure()
                    plot_confusion_matrix(cnf_matrix, classes=confusion_matrix_labels,
                                          title='Confusion matrix, without normalization')

                    plt.show()
                    plt.savefig(cnf_file,dpi=300);
                    if(step % 300 ==0):
                        confusion_matrix_gt_labels = np.array([]);
                        confusion_matrix_predictions = np.array([]);

    sess.close()


from bs4 import BeautifulSoup as bs
from urllib.request import urlopen
from flask_caching import Cache
import time
import yaml
import os
import requests
import re
import pickle
import sys
import html
from datetime import datetime
from flask import Flask, render_template, flash, request, redirect, url_for, session, logging
from flask_paginate import Pagination, get_page_parameter
from wtforms import Form, StringField

DEBUG = True
FETCH_DELAY = 0

def get_media_imdb(imdbid):
	if DEBUG:
		print('------ get imdb data ------- sleep {} s'.format(FETCH_DELAY))

	new = True
	url = 'http://www.imdb.com/title/{}/'.format(imdbid)
	try:
		soup = bs(urlopen(url), 'html.parser')
		time.sleep(FETCH_DELAY)
	except:
		return None, False
	# parse rating
	ss = soup.findAll('span',attrs={'itemprop':'ratingValue'})
	try:
		rating = str(ss).split(">")[1].split("<")[0]
	except:
		rating = '0'
	# parse poster
	try:
		poster = soup.findAll('div',attrs={'class':'poster'})
		poster = str(poster).split('"')[9]
	except:
		poster = MISSING_IMDB_POSTER
	# parse genre
	gen = soup.findAll('div',attrs={'class':'subtext'})
	x = bs(str(gen), "html.parser")
	gen = x.findAll('span',attrs={'class':'itemprop'})
	genre = re.sub('<[^<]+?>', '', str(gen))
	genre = genre.strip('[').strip(']').strip(' ').split(",")

	# parse stars
	stars = soup.findAll('span',attrs={'itemprop':'actors'})
	stars = (re.sub('<[^<]+?>', '', str(stars)).replace('\n', '')
												.replace('[', '')
												.replace(']', '')
												.replace('             ', '')
												.replace(',, ',',')
												.rstrip()
												.split(',')
												)
	stars = list(filter(None, stars))

	# parse title
	t = soup.findAll('meta', attrs = {'property': 'og:title'})
	title = html.unescape(str(t).split('"')[1])
	if '(' in title:
		title = title.split('(')[0]
	if len(title) < 1:
		return None, False
	# parse year
	try:
		type = 'movie'
		title_year = soup.find('span', {'id': 'titleYear'}).find('a').get_text()
	except:
		type = 'serial'
		try:
			title_year = soup.find('a', {'title': 'See more release dates'})
			title_year = title_year.get_text().strip().split('(')[1].split(')')[0]
		except:
			try:
				title_year = str(soup.find('span', {'class': 'parentDate'})
											.get_text()
											.strip()
											)
				title_year = title_year.replace('(', '').replace(')', '')
			except:
				title_year = None

	genre_str = ', '.join(genre)
	# print(genre_str)
	result = {'Imdb_id': imdbid, 'Type': type,
				'Year': title_year, 'Title': title,
				'imdbRating': rating, 'Poster': poster,
				'Genre': genre, 'Genre_str': genre_str,
				'stars': stars,
				}

	return result, new

if __name__ == '__main__':
	print(get_media_imdb('tt1856101'))


import requests
import bs4
from time import sleep
import random
import json

base_url = "https://www.rekrute.com"

random_delays = [6, 11, 7, 10, 9, 13, 15, 8, 12]

DB_FILE_LINK = "./src/db_rekrute.json"

headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}

def get_pages(first_page_link):
    response = requests.get(first_page_link, headers=headers)
    html_doc = response.text
    soup = bs4.BeautifulSoup(html_doc, 'html.parser')
    pages_links_container = soup.find("span", {"class": "jobs"})
    pages_links_select = pages_links_container.find("select")
    print("Found {} pages".format(len(pages_links_select)))
    return pages_links_select.find_all("option")

def extract_job_description(href):
    response = requests.get(f"{base_url}{href}", headers=headers)
    html_doc = response.text
    soup = bs4.BeautifulSoup(html_doc, 'html.parser')
    poste_container = soup.find("h2",string="Poste :")
    profil_recherche_container = soup.find("h2",string="Profil recherché :")
    poste_container_adjacent_container = poste_container.parent.text
    profil_recherche_adjacent_container = profil_recherche_container.parent.text
    return [poste_container_adjacent_container, profil_recherche_adjacent_container]

def get_jobs_data(link):
    print(link)
    response = requests.get(f"{base_url}{link}")
    html_doc = response.text
    soup = bs4.BeautifulSoup(html_doc, "html.parser")
    jobs = soup.find_all("li", {"class" : "post-id"})
    jobs_arr = []
    for job in jobs:
        job_obj = dict()
        text = job.find("a", {"class" : "titreJob"})
        data_date_container = job.find("em", {"class" : "date"})
        date_data = data_date_container.find_all("span")
        link_href = text['href']
        job_description = extract_job_description(link_href)

        job_obj["poste"] = remove_spaces_noise(job_description[0])
        job_obj["profil_recherche"] = remove_spaces_noise(job_description[1])
        infos = job.find_all("li")

        raw_data = {
            'job_secteur_activite' : infos[0],
            'job_fonction' : infos[1],
            'job_experience_requise' : infos[2],
            'job_niveau_etude_demande' : infos[3],
            'job_contrat_propose' : infos[4],
        }

        for value in raw_data:
            raw_elm = raw_data[value]
            if value == 'job_fonction':
                raw_elements = raw_elm.find_all("a")
                property_elements = []
                for re in raw_elements:
                    property_elements.append(re.text.strip())
                job_obj[value] = property_elements
            else:
                raw_element = raw_elm.find("a")
                job_obj[value] = raw_element.text.strip()

        if text:
            job_obj['title'] = text.text.strip()
        if len(date_data) > 0 and date_data[0]:
            job_obj['date_publication'] = date_data[0].text
        if len(date_data) >= 2 and date_data[1]:
            job_obj['date_expiration'] = date_data[1].text
        if len(date_data) >= 3 and date_data[2]:
            job_obj['nombre_postes'] = date_data[2].text
        # job_obj["activites"] = job_activites

        jobs_arr.append(job_obj)

    with open(DB_FILE_LINK, "r", encoding="utf-8") as read_file:
        file_json_data = json.load(read_file)
        final_arr = file_json_data + jobs_arr
        with open(DB_FILE_LINK, "w", encoding="utf-8") as write_file:
            json.dump(final_arr, write_file)
        write_file.close()

    delay()

def delay():
    # start delay before next request (IMPORTANT to not get IP banned)
    rnd_delay = random.choice(random_delays)
    print(f"Delaying for {rnd_delay} seconds ...")
    sleep(rnd_delay)

def remove_spaces_noise(text):
    return text.replace("\n","").replace("\r", "").replace("\t", "").strip()


#!/usr/bin/env python3
import os, sys, getopt, logging

from FaqBot.Command import Command

from telegram.ext import Application, CommandHandler, MessageHandler

from pathlib import Path

# Enable logging
logging.basicConfig(
    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=logging.INFO
)

logger = logging.getLogger(__name__)

# This contains the list of all commands: `/command`
# stores objects as FaqBot.Command
commands = {}

# Iterate over all command directories, and create a FaqBot.Command for each
# This in turn will iterate over all files to gather their respective FaqBot.Entrys
def loadFiles(basedir) -> None:
    base = Path(basedir)
    for cmd in base.iterdir():
        if not cmd.is_dir():
            continue
        commands[cmd.name] = Command(cmd)
    return

# Debug: Log every message
async def logMessages(update, context):
    print(update.message.text)

# This is our main entry point
def main(argv) -> None:
    # First order of business is argument parsing. We use getopt for that
    basedir = ''
    tokenfile = ''

    try:
        opts, args = getopt.getopt(argv, "b:t:", ["basedir=", "token="]);
    except getopt.GetoptError:
        print(sys.argv[0] + " -d path/to/faq-basedir -t tokenfile")
        sys.exit(1)

    for opt, arg in opts:
        if opt in ("-b", "--basedir"):
            basedir = arg
        elif opt in ("-t", "--token"):
            tokenfile = arg

    # Make sure we got all relevant parameters
    if basedir == '':
        print("Please pass a basedir using -b")
        sys.exit(1)

    if tokenfile == '':
        print("Please pass a tokenfile using -t")
        sys.exit(1)

    # Load the Commands + their Entries
    loadFiles(basedir)

    # Open the token file (the intention is to hide the token from the command line)
    tokenF = open(tokenfile, "r")
    token = tokenF.read().replace("\n", "")

    # Prepare the actual bot
    app = Application.builder().token(token).build()

    #app.add_handler(MessageHandler(None, logMessages, block=False))
    # register the handler for each `/cmd`
    for cmd in commands:
        app.add_handler(CommandHandler(cmd, commands[cmd].handler))

    # And start the actual bot.
    # We just quit on ctrl+d (or if the unit file tells us to)
    app.run_polling()

# Start the actual program
if __name__ == '__main__':
    main(sys.argv[1:])


# -*- encoding: utf-8 -*-
"""
@File    :   re_msie_egp_model.py
@Time    :   2023/02/02 11:13:57
@Author  :   jiangjiajia
"""
import copy

import numpy as np
import torch

from ..metrics.metric import compute_f1
from ..modules.efficient_globalpointer import EfficientGlobalPointer
from ..modules.multi_label_crossentropy import MultiLabelCrossentropy
from ..utils.common_utils import (ENCODERS, logger_output, read_jsonline)
from .base_model import BaseModel

class REMSIEEGPModel(BaseModel):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        encoder = ENCODERS.get(self.encoder.get('type', ''), None)
        if not encoder:
            logger_output('error', 'encoder type wrong or not configured')
            raise ValueError('encoder type wrong or not configured')
        self.encoder = encoder.from_pretrained(self.encoder.get('pretrained_model_dir', ''))
        # efficient globalpointer
        self.egp = EfficientGlobalPointer(self.encoder.config.hidden_size, self.num_labels, self.hidden_size)
        self.criterion = MultiLabelCrossentropy(reduction='mean')
        # self.criterion = torch.nn.BCEWithLogitsLoss(reduction='mean')

        # 读取Schema，用于预测结果过滤
        schema_info = read_jsonline(self.schema_path)
        self.schema = {}
        for each_schema in schema_info:
            subject_type = each_schema['subject_type']
            predicate = each_schema['predicate']
            object_type = each_schema['object_type']
            self.schema.setdefault(predicate, set())
            self.schema[predicate].add(subject_type + '|' + object_type)

    def forward(self, input_ids, token_type_ids, attention_mask, labels=None, label_mask=None, phase=None):
        encoder_outputs = self.encoder(input_ids=input_ids, token_type_ids=token_type_ids,
                                       attention_mask=attention_mask).last_hidden_state
        # efficient globalpointer
        logits = self.egp(encoder_outputs)
        if phase == 'train' and labels is not None and label_mask is not None:
            logits = logits[label_mask == 1].view(-1, self.num_labels)
            labels = labels[label_mask == 1].view(-1, self.num_labels)
            # logits = logits - (1 - label_mask.unsqueeze(-1).expand(-1, -1, -1, logits.shape[-1])) * 1e12
            loss = self.criterion(logits, labels)
            return {
                'loss': loss,
                'logits': logits
            }
        else:
            return {
                'logits': logits
            }

    def get_predictions(self, forward_output, forward_target, dataset, batch_start_index=0):
        predictions = []
        idx = 0

        for each_output in forward_output['logits']:
            target_data = copy.deepcopy(dataset.data[idx + batch_start_index])
            idx += 1
            text = target_data['text']
            entity_dict = {}
            head2head_set = set()
            tail2tail_set = set()
            for i, j, z in zip(*np.where(each_output > 0)):
                if i == j:
                    continue
                start_index = i - 1
                end_index = j - 1
                label_type, label = dataset.id2label[z].split('_')
                if label_type == 'EH2ET':
                    if end_index < start_index:
                        continue
                    entity = (text[start_index:end_index + 1], start_index, end_index, label)
                    entity_dict.setdefault(start_index, []).append(entity)
                elif label_type == 'H2H':
                    head2head_set.add((start_index, end_index, label))
                elif label_type == 'T2T':
                    tail2tail_set.add((start_index, end_index, label))

            relations_dict = []
            for start_index, end_index, label in head2head_set:
                sub_entity_list = entity_dict.get(start_index, [])
                obj_entity_list = entity_dict.get(end_index, [])
                for sub in sub_entity_list:
                    for obj in obj_entity_list:
                        sub_text, sub_start_index, sub_end_index, sub_label = sub
                        obj_text, obj_start_index, obj_end_index, obj_label = obj
                        if (sub_end_index, obj_end_index, label) in tail2tail_set:
                            relations_dict.append((sub, label, obj))
            spo_list = []
            for sub, label, obj in relations_dict:
                subject = sub[0]
                subject_type = sub[3]
                predicate = label
                object = obj[0]
                object_type = obj[3]
                if subject_type + '|' + object_type not in self.schema[predicate]:
                    continue
                spo_list.append({
                    'subject': subject,
                    'subject_type': subject_type,
                    'predicate': predicate,
                    'object': {'@value': object},
                    'object_type': {'@value': object_type}
                })
            target_data['spo_list'] = spo_list
            predictions.append(target_data)
        return predictions

    def get_metrics(self, phase, predictions, dataset):
        def get_relations(data):
            relations = []
            entities = []
            for each_data in data:
                spo_list = each_data['spo_list']
                for each_spo in spo_list:
                    subject = each_spo['subject']
                    subject_type = each_spo['subject_type']
                    predicate = each_spo['predicate']
                    object = each_spo['object']['@value']
                    object_type = each_spo['object_type']['@value']
                    entities.append((subject, subject_type))
                    entities.append((object, object_type))
                    relations.append((subject, subject_type, predicate, object, object_type))
            return relations, entities

        predictions, pred_entities = get_relations(predictions)
        targets, target_entities = get_relations(dataset.data)
        results = {}
        results['F1'] = compute_f1(predictions, targets)['F1']
        results['entity_F1'] = compute_f1(pred_entities, target_entities)['F1']
        logger_output('info', 'metrics F1:{}'.format(results['F1']))
        return results

model = REMSIEEGPModel