from __future__ import print_function, division
import pandas as pd
import numpy as np
# import matplotlib
# matplotlib.use('qt4agg')
import matplotlib.pyplot as plt
from sklearn.preprocessing import MinMaxScaler, minmax_scale
def mae(y, prediction):
    return np.mean(np.abs(y - prediction))

# Read in full data set
X = pd.read_csv(
    'Bike-Sharing-Dataset/hour.csv', sep=',',
    dayfirst=False,
    # parse_dates={'datetime' : ['dteday', 'hr']},
    #names=['yr', 'mnth', 'holiday', 'workingday', 'weathersit', 'weekday', 'temp', 'atemp', 'hum', 'windspeed']
    )

# Add time series
X = X.sort_values(['dteday', 'hr'])
# Add ewma over same hour
cols = ['hr', 'weekday']
#cols = ['hr']
gb = X[['cnt'] + cols].groupby(cols)

for halflife in [1.5]:
    def ewma(X):
        X = X.shift()
        X = pd.DataFrame(pd.ewma(X['cnt'], halflife=halflife))
        return X
    x = gb.transform(ewma)
    x = x.fillna(0)
    print("MAE ewma", halflife, mae(X['cnt'], x['cnt']))

#X['cnt_ewma'] = x['cnt']
#from IPython import embed
#embed()

def train_test_split(X):
    n = len(X)
    index = np.random.permutation(np.arange(0, n))

    X_train = X.iloc[index[n // 10:]]
    X_test = X.iloc[index[:n // 10]]
    assert len(X) == len(X_train) + len(X_test)
    X_train = X_train.reset_index(drop=True)
    X_test = X_test.reset_index(drop=True)
    return X_train, X_test

df_train, df_test = train_test_split(X)

print("MAE trivial", mae(df_test['cnt'], np.mean(df_train['cnt'])))
print("MAE trivial median", mae(df_test['cnt'], np.median(df_train['cnt'])))

from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import GradientBoostingRegressor
est = RandomForestRegressor(n_estimators=2000, n_jobs=4)
#est = GradientBoostingRegressor(n_estimators=2000)
def extract_target(X):
    y = np.asarray(X['cnt'])
    Xnp = X.copy()
    del Xnp['cnt']
    del Xnp['dteday']
    del Xnp['casual']
    del Xnp['registered']
    del Xnp['instant']
    columns = Xnp.columns
    Xnp = np.asarray(Xnp)
    return Xnp, y, columns

X_train, y_train, columns = extract_target(df_train)
#est.fit(X_train, y_train - df_train['cnt_ewma'])
est.fit(X_train, y_train)

X_test, y_test, _ = extract_target(df_test)

prediction = est.predict(X_test)
#prediction += df_test['cnt_ewma']
importances = pd.Series(est.feature_importances_, index=columns)
importances.plot(kind='bar')
plt.show()
print("MAE trivial random forest", mae(df_test['cnt'], prediction))

# fit
df_test['pred'] = prediction
gb = df_test.groupby('dteday').mean()
gb['cnt'] .plot()
gb['pred'].plot()
plt.show()
sm = gb
sm['cnt_day'] = sm['cnt']
sm = sm.reset_index()

# predict
Xm = X.merge(sm[['cnt_day', 'dteday']], on=['dteday'], how='left')


import os
from datetime import datetime
import re
import logging
from os.path import join

def validate_date_prefix(date_prefix, formats, present):
    '''Returns True if the string date_prefix is a valid date, given the list
    of formats and if the date is smaller than the given present date.
    '''

    for format in formats:
        try:
            file_date = datetime.strptime(date_prefix, format)
            return file_date.date() < present.date()
        except Exception:
            pass
    logging.info(date_prefix + " is not a valid date to format "
                 + "for all formats or its in the future.")
    return False

def match_to_rename(reg, re_subs, re_date, file_name, formats, present):
    '''Verifies if the file_name matches to a valid date, that could not be
    greater than the present date given the list of formats of date.
    Returns the groups of matched regex substitution or None if do not match.
    '''

    if re.match(reg, file_name):
        date_prefix = re.sub(reg, re_date, file_name)
        if validate_date_prefix(date_prefix, formats, present):
            return re.sub(reg, re_subs, file_name)

def format_name(file_name, present):
    '''Returns formatted name if matches to a date or None.

    Parameters:
        file_name (str): The name of the file to be renamed
        present (datetime): The date on which the name cannot pass. Ex.: today

    Returns:
        formatted_name (str): A new str, formatted like '%Y-%m-%d_%H%M%S' if
        the initial name contains date and hour or '%Y-%m-%d' if it just
        contains date or None if does not match.
    '''

    # File name can contain prefix (ex: IMG, VID_)
    prefix = r"^(IMG|VID|Screenshot(\ from)?|InShot|PANO)?"
    # File name should have a valid date (%Y-%m-%d or %Y%m%d)
    date = r"(_|-|\ )?(\d{4})-?(\d{2})-?(\d{2})"
    # File name can contain suffix for date hours (ex: _293823)
    hour = r"(_|-|\ )?(\d{2})-?(\d{2})-?(\d{2})"
    # Suffix of file name and ext.
    suffix = r"(.*)?"

    date_hour_formats = ['%Y-%m-%d_%H%M%S',
                         '%Y-%m-%d_%H-%M-%S',
                         '%Y-%m-%d-%H-%M-%S',
                         '%Y-%m-%d %H-%M-%S']
    formatted_name = match_to_rename(reg=prefix+date+hour+suffix,
                                     re_subs=r"\4-\5-\6_\8\9\10\11",
                                     re_date=r"\4-\5-\6_\8\9\10",
                                     file_name=file_name,
                                     formats=date_hour_formats,
                                     present=present)

    # If the file name don't contain date and hour, it can contain only date
    if not formatted_name:
        formatted_name = match_to_rename(reg=prefix+date+suffix,
                                         re_subs=r"\4-\5-\6\7",
                                         re_date=r"\4-\5-\6",
                                         file_name=file_name,
                                         formats=['%Y-%m-%d'],
                                         present=present)

    # If not formatted both on date and hour or just date, it will be None
    return formatted_name

def preview_rename_file(file, present):
    old_name = os.path.split(os.path.abspath(file))[1]
    new_name = format_name(old_name, present)
    return [old_name, new_name]

def preview_rename_all(files):
    present = datetime.now()
    new_names = []
    preview_renamed_files = []
    will_not_be_renamed = []
    for name in sorted(files):
        names = preview_rename_file(name, present)
        if names:
            old_name = names[0]
            new_name = names[1]
            if new_name not in new_names:
                new_names.append(new_name)
                preview_renamed_files.append([old_name, new_name])
            else:
                will_not_be_renamed.append([old_name, new_name])
    return preview_renamed_files, will_not_be_renamed

def rename_file(file, final_path, old_name, new_name):
    '''Renames (and moves) the file to the final_path.
    Only renames if the file begins with prefix+date_hour or prefix+date
    and if a file with de same name already doesn't exists on destiny.
    '''
    # Only renames if the file begins with prefix+date_hour or prefix+date
    # and if a file with de same name already doesn't exists on destiny.
    try:
        formatted_path_file = os.path.join(final_path, new_name)
        if (not (os.path.exists(formatted_path_file))
           and file != formatted_path_file):
            os.rename(file, formatted_path_file)
            logging.info("| " + old_name + " | " + new_name + " |")
            return new_name
        else:
            # TODO Compare metadata of files
            logging.error("Renamed file " + old_name + " already exists"
                          + " in destiny directory. Nothing to do.")
    except Exception as e:
        logging.info("| " + old_name + " | ERROR: " + str(e) + " |")
        logging.error(e)

def rename_all(directory, final_path, files):
    '''Renames the files, removing the prefix like "IMG_", "VID-" and
    formatting the new date with hyphens. Ex. 2020-10-06_22-10-10.png.
    Returns the list of modified files new names.
    '''
    files = sorted(files)
    preview_renamed_files, not_renamed_files = preview_rename_all(files)

    print('These files will NOT be renamed:')
    print(not_renamed_files)

    print('These files will be renamed:')
    print(*preview_renamed_files, sep='\n')

    if len(preview_renamed_files) > 0:
        answer = input('Would you like to rename these '
                       + str(len(preview_renamed_files)) + ' files?'
                       + ' Type N for NO and any other for YES.\n')

        if answer.lower() != 'n':
            logging.info("Renaming all files...")
            logging.info("|     Name     |      New Name     |")
            renamed_files = []

            for old_name, new_name in preview_renamed_files:
                renamed = rename_file(join(directory, old_name),
                                      final_path, old_name, new_name)
                if renamed:
                    renamed_files.append(renamed)
            print(str(len(renamed_files)) + ' files renamed.')
            logging.info(str(len(renamed_files)) + " files renamed.")

            return renamed_files


#!/usr/bin/env python

# Import PyGame library
import pygame
import adapter
import MD3Buttons
import RetroflagSnes
import ArduinoMicro
from actions import *

import logger

controllers = { MD3Buttons, RetroflagSnes, ArduinoMicro, }

pad = {}
joysticks = {}

currentGuid = ""
currentAdapter = -1

def pressButton(btn):
  try:
    logger.debug(f"Button {btn} pressed.")
    btnValue = pad[btn]
    adapter.pressButton(btnValue)
  except KeyError:
    logger.debug("Button not managed")

def releaseButton(btn):
  try:
    logger.debug(f"Button {btn} released.")
    btnValue = pad[btn]
    adapter.releaseButton(btnValue)
  except KeyError:
    logger.debug("Button not managed")

def manageAxis(axis, value):
  logger.debug(f"Axis change: axis: {axis} value: {value}")
  axisValue = f"{axis}_{value}"
  try:
    btnValue = pad[axisValue]
    adapter.performAction(btnValue)
  except KeyError:
    logger.debug("Axis not managed.")

def manageHat(hat, value):
  logger.debug(f"Hat change: hat: {hat} value: {value}")
  try:
    first = value[0]
    second = value[1]

    fisrtHatValue = f"{hat}_1_{first}"
    secondHatValue = f"{hat}_2_{second}"

    btnValue = pad[fisrtHatValue]
    adapter.performAction(btnValue)

    btnValue = pad[secondHatValue]
    adapter.performAction(btnValue)
  except KeyError:
    logger.debug("Hat not managed.")

def checkConnections():
  global currentAdapter
  global currentGuid
  foundGuid = False
  if currentAdapter == adapter.NOT_INITIALIZED:
    logger.info(f"Adapter not initialized")
    pad.clear()
  elif currentAdapter == adapter.NOT_CONNECTED:
    logger.info(f"No adapter connected")
    pad.clear()
  elif currentGuid == "":
    logger.info(f"No controller connected")
    pad.clear()
  else:
    for controller in controllers:
      if controller.guid == currentGuid:
        foundGuid = True
        logger.info(f"{controller.name} connected")
        mapController(controller)
    if foundGuid == False:
      logger.info(f"Unknown controller")
      pad.clear()

def mapController(controller): # Add try / catch if adapter not defined for controller
  pad.clear()
  try:
    if currentAdapter == adapter.MEGADRIVE:
      logger.info(f"Map for Sega Megadrive")
      pad.update(controller.sega_md)
    elif currentAdapter == adapter.SUPER_NES:
      logger.info(f"Map for Super Nintendo")
      pad.update(controller.snes)
    elif currentAdapter == adapter.SATURN:
      logger.info(f"Map for Sega Saturn")
      pad.update(controller.saturn)
    else:
      logger.info(f"Unknown adapter")
  except AttributeError:
    logger.warning(f"Controller not usable with this adapter")

def main():
  global currentAdapter
  global currentGuid
  logger.info("!!!!!!!!!!!!!! START !!!!!!!!!!!!!!")

  adapter.init()
  currentAdapter = adapter.currentAdapter

  # Initialize Joystick(s).
  pygame.init()

  while True:
    for event in pygame.event.get():
      if event.type == pygame.JOYHATMOTION:
        manageHat(event.hat, event.value)

      if event.type == pygame.JOYAXISMOTION:
        manageAxis(event.axis, round(event.value))

      if event.type == pygame.JOYBUTTONDOWN:
        pressButton(f"{event.button}")

      if event.type == pygame.JOYBUTTONUP:
        releaseButton(f"{event.button}")

      if event.type == pygame.JOYDEVICEADDED:
          joy = pygame.joystick.Joystick(event.device_index)
          joysticks[joy.get_instance_id()] = joy
          name = joy.get_name()
          logger.debug(f"Joystick {joy.get_instance_id()} connected\nname: {name}, id: {joy.get_id()}, guid: {joy.get_guid()}\naxes: {joy.get_numaxes()}, buttons: {joy.get_numbuttons()}, balls: {joy.get_numballs()}, hats: {joy.get_numhats()}")
          currentGuid = joy.get_guid()
          checkConnections()

      if event.type == pygame.JOYDEVICEREMOVED:
        del joysticks[event.instance_id]
        logger.debug(f"Joystick {event.instance_id} disconnected")
        currentGuid = ""
        checkConnections()

    pluggedAdapter = adapter.currentAdapter
    if pluggedAdapter != currentAdapter:
      logger.debug(f"Adapter status changed: {pluggedAdapter}")
      currentAdapter = pluggedAdapter
      checkConnections()

main()


from django.contrib import admin
from django.urls import path, include
from .views import register_view, login_view, index, profile_view, oikawakuroo

urlpatterns = [
    path('home/', index, name='home'),
    path('register/', register_view, name='register'),
    path('login/', login_view, name='login'),
    path('profile/', profile_view, name='profile'),
    path('oikawakuroo/',oikawakuroo, name = 'oikawakuroo' )
]


import cocotb
from cocotb.triggers import Timer, RisingEdge, FallingEdge, ClockCycles
from cocotb.result import TestFailure
from cocotb.clock import Clock
from cocotb.binary import BinaryValue
from cocotb.scoreboard import Scoreboard
from cocotb.monitors import Monitor
from cocotb.regression import TestFactory
from cocotb.drivers.amba import AXI4LiteMaster
from cocotb.drivers.amba import AXIProtocolError
from cocotb.wavedrom import trace, Wavedrom
from cocotb import SimLog, coroutine
from cocotb.drivers import BitDriver

import random
import numpy as np
import math
import sys, os
import matplotlib.pyplot as plt

# Constants
c_BASEADDRESS = 0x00000000
c_VERSION_OFFSET = 0x00000000
c_CONFIG_ID_OFFSET = 0x00000004
c_COUNT_OFFSET = 0x00000008
c_VERSION_VALUE = 0x00000001
c_CONFIG_ID_VALUE = 0x00000001
c_COUNT_VALUE = 0x02000000
c_CLK_PERIOD = 10 #ns
# ==============================================================================
class BitMonitor(Monitor):
    """Observe a single-bit input or output of the DUT."""
    def __init__(self, name, signal, clk, callback=None, event=None):
        self.name = name
        self.signal = signal
        self.clk = clk
        Monitor.__init__(self, callback, event)

    @coroutine
    def _monitor_recv(self):
        clkedge = RisingEdge(self.clk)

        while True:
            # Capture signal at rising edge of clock
            yield clkedge
            vec = self.signal.value
            self._recv(vec)
# ==============================================================================
class outMonitor(Monitor):
    def __init__(self, name, dut, callback=None, event=None):
        self.name = name
        self.dut = dut
        Monitor.__init__(self, callback, event)

    @cocotb.coroutine
    def _monitor_recv(self):
        clkedge = RisingEdge(self.dut.clk)
        while True:
            # Capture signal at rising edge of clock
            yield clkedge
            vec = self.dut.Zybo_Example_leds_rgb_out
            self._recv(vec)
# ==============================================================================
def input_gen():
    """Generator for input data applied by BitDriver.
    Continually yield a tuple with the number of cycles to be on
    followed by the number of cycles to be off.
    """
    while True:
        yield random.randint(1, 5), random.randint(1, 5)
# ==============================================================================
class TB(object):
    def __init__(self, dut):
        # Some internal state
        self.dut = dut
        self.stopped = False

        # Use the input monitor to reconstruct the transactions from the pins
        # and send them to our 'model' of the design.
        self.input_mon = BitMonitor(name="input", signal=dut.Zybo_Example_sw_in, clk=dut.clk,
                                    callback=self.model)

        # Create input driver and output monitor
        self.input_drv = BitDriver(signal=dut.Zybo_Example_sw_in, clk=dut.clk, generator=input_gen())
        self.output_mon = BitMonitor(name="output", signal=dut.Zybo_Example_leds_out, clk=dut.clk)

        # Create a scoreboard on the outputs
        self.expected_output = []
        self.scoreboard = Scoreboard(dut)
        self.scoreboard.add_interface(self.output_mon, self.expected_output)

    def model(self, transaction):
        if not self.stopped:
            self.expected_output.append(transaction)

    def start(self):
        self.input_drv.start()

    def stop(self):
        self.input_drv.stop()
        self.stopped = True
# ==============================================================================
def check(dut, act, exp):
    # Check
    if act != exp:
        raise TestFailure("Register at address 0x%08X should have been: \
                               0x%08X but was 0x%08X" % (c_BASEADDRESS+c_COUNT_OFFSET, c_COUNT_VALUE, int(act)))

    dut._log.info("Read: 0x%08X From Address: 0x%08X" % (int(act), c_BASEADDRESS+c_COUNT_OFFSET))
# ==============================================================================
@cocotb.test(skip = False, stage = 1)
def Zybo_Example_test(dut):
    # Setting up clocks
    clk_100MHz = Clock(dut.clk, c_CLK_PERIOD, units='ns')
    cocotb.fork(clk_100MHz.start(start_high=False))
    axi_aclk_100MHz = Clock(dut.axi_aclk, c_CLK_PERIOD, units='ns')
    cocotb.fork(axi_aclk_100MHz.start(start_high=False))

    # Setting init values
    dut.reset = 1
    dut.Zybo_Example_sw_in = 0
    dut.Zybo_Example_bt_in = 0
    dut.axi_aresetn = 0
    # AXI-Lite Master object
    axil_m = AXI4LiteMaster(dut, "s_axi", dut.axi_aclk)
    # tb
    tb=TB(dut)
    tb.start()
    # Wait one cycle and deactivate resets
    yield Timer(c_CLK_PERIOD, units='ns')
    dut.reset <= 0
    dut.axi_aresetn <= 1
    yield Timer(c_CLK_PERIOD, units='ns')

    # AXI-Lite read VERSION
    dut._log.info("AXI-Lite: Reading address 0x%02X" % (c_BASEADDRESS+c_VERSION_OFFSET))
    s_value_read = yield axil_m.read(c_BASEADDRESS+c_VERSION_OFFSET)
    # Check
    check(dut, s_value_read, c_VERSION_VALUE)

    # AXI-Lite read CONFIG_ID
    dut._log.info("AXI-Lite: Reading address 0x%02X" % (c_BASEADDRESS+c_CONFIG_ID_OFFSET))
    s_value_read = yield axil_m.read(c_BASEADDRESS+c_CONFIG_ID_OFFSET)
    # Check
    check(dut, s_value_read, c_CONFIG_ID_VALUE)

    # AXI-Lite read COUNT
    dut._log.info("AXI-Lite: Reading address 0x%02X" % (c_BASEADDRESS+c_COUNT_OFFSET))
    s_value_read = yield axil_m.read(c_BASEADDRESS+c_COUNT_OFFSET)
    # Check
    check(dut, s_value_read, c_COUNT_VALUE)

    # AXI-Lite write
    yield axil_m.write(c_BASEADDRESS+c_COUNT_OFFSET, 0x00000001)
    yield Timer(c_CLK_PERIOD, units='ns')

    # end tb
    tb.stop()

@cocotb.test(skip = False, stage = 2)
def Wavedrom_test(dut):
    # Setting up clocks
    clk_100MHz = Clock(dut.clk, c_CLK_PERIOD, units='ns')
    cocotb.fork(clk_100MHz.start(start_high=False))
    axi_aclk_100MHz = Clock(dut.axi_aclk, c_CLK_PERIOD, units='ns')
    cocotb.fork(axi_aclk_100MHz.start(start_high=False))

    # AXI-Lite Master object
    axil_m = AXI4LiteMaster(dut, "s_axi", dut.axi_aclk)

    # Setting init values
    dut.reset <= 1
    dut.Zybo_Example_sw_in <= 4
    dut.Zybo_Example_bt_in <= 0
    dut.axi_aresetn <= 0
    # Wait one cycle and deactivate resets
    yield Timer(c_CLK_PERIOD, units='ns')
    dut.reset <= 0
    dut.axi_aresetn <= 1
    yield Timer(c_CLK_PERIOD, units='ns')

    # AXI-Lite write
    yield axil_m.write(c_BASEADDRESS+c_COUNT_OFFSET, 0x00000001)
    yield Timer(c_CLK_PERIOD, units='ns')

    # Wavedrom
    args = [dut.Zybo_Example_sw_in, dut.Zybo_Example_leds_out, dut.Zybo_Example_leds_rgb_out]
    with trace(*args, clk=dut.clk) as waves:
        yield ClockCycles(dut.clk, 12)
        dut._log.info(waves.dumpj(header = {'text':'WaveDrom example', 'tick':0}))
        waves.write('wavedrom.json', header = {'tick':0}, config = {'hscale':3})


# fixtures are functions that are run by the pytest before the actual test functions
# eg= setup database connection, initialize webdriver
import pytest

# define fixtures function (parent method)
@pytest.fixture()
def setup_list():
    print("\n in fixtures..\n")
  #  city2 = ["new york", "london", "mumbai"]
    city = ["new york","london","mumbai"]
    return city

# getting data from setup_list function , inherits its properties
def test_getItem(setup_list):  # (child method1)
    print(setup_list[1:3])
    assert setup_list[0] == "new york"
    assert setup_list[::2] == ["new york", "mumbai"]


import functools

from keras.layers import *
from keras.models import Model
from keras.wrappers.scikit_learn import KerasClassifier
from keras.regularizers import l2
from keras.optimizers import SGD
import numpy as np
import pandas as pd
import tensorflow as tf
import os
import pickle
import plac

from rectified_adam import RectifiedAdam

from sklearn.ensemble import AdaBoostClassifier
from sklearn.metrics import roc_auc_score

from cfg import *

OPTIMIZER = RectifiedAdam(lr=LR, decay=LR / (EPOCHS * BATCHES))

# OPTIMIZER = SGD(lr=LR, decay=LR/(EPOCHS*BATCHES), momentum=0.9)

def fix_tf():
    # Fix Tensorflow Memory Usage
    gpus = tf.config.experimental.list_physical_devices('GPU')
    if gpus:
        try:
            # Currently, memory growth needs to be the same across GPUs
            for gpu in gpus:
                tf.config.experimental.set_memory_growth(gpu, True)
            logical_gpus = tf.config.experimental.list_logical_devices('GPU')
            print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs")
        except RuntimeError as e:
            # Memory growth must be set before GPUs have been initialized
            print(e)

def build_keras():
    ip = Input((INPUT_DIMS,))

    def f_lambda_smiles(x):
        x = x[:, SMILES_IDX_I:SMILES_IDX_F]
        x = tf.reshape(x, (-1, SMILES_LEN, SMILES_DIMS))
        return x

    def f_lambda_lines(x):
        return x[:, LINES_IDX_I:LINES_IDX_F]

    def f_lambda_maccs(x):
        return x[:, MACCS_IDX_I:MACCS_IDX_F]

    def f_lambda_target(x):
        return x[:, TARGET_IDX_I:TARGET_IDX_F]

    lambda_smiles = functools.partial(f_lambda_smiles)
    lambda_smiles.__name__ = 'lambda_smiles'
    lambda_lines = functools.partial(f_lambda_lines)
    lambda_lines.__name__ = 'lambda_lines'
    lambda_maccs = functools.partial(f_lambda_maccs)
    lambda_maccs.__name__ = 'lambda_maccs'
    lambda_target = functools.partial(f_lambda_target)
    lambda_target.__name__ = 'lambda_target'

    features = []

    # Cell Lines
    l_dropout = Dropout(LINES_DROPOUT)

    # forward
    lx = Lambda(lambda x: lambda_lines(x))(ip)
    if LINES_DROPOUT > 0:
        lx = l_dropout(lx)
    features.append(lx)

    if SMILES_ENABLE:
        s_conv1 = Conv1D(SMILES_KERNEL_DIMS, 11, padding='same',
                         kernel_regularizer=l2(SMILES_L2),
                         kernel_initializer=SMILES_KERNEL_INIT)
        s_bn1 = BatchNormalization()
        s_relu1 = ReLU()
        s_ap1 = AveragePooling1D(pool_size=5)
        s_conv2 = Conv1D(SMILES_KERNEL_DIMS, 11, padding='same',
                         kernel_regularizer=l2(SMILES_L2),
                         kernel_initializer=SMILES_KERNEL_INIT)
        s_bn2 = BatchNormalization()
        s_relu2 = ReLU()
        s_ap2 = AveragePooling1D(pool_size=5)
        s_gmp = GlobalMaxPooling1D()

        # forward
        sx = Lambda(lambda x: lambda_smiles(x))(ip)
        sx = s_conv1(sx)
        sx = s_bn1(sx)
        sx = s_relu1(sx)
        sx = s_ap1(sx)
        sx = s_conv2(sx)
        sx = s_bn2(sx)
        sx = s_relu2(sx)
        sx = s_ap2(sx)
        sx = s_gmp(sx)

        features.append(sx)

    # MACCS
    if MACCS_ENABLE:
        m_dropout = Dropout(LINES_DROPOUT)

        # forward
        mx = Lambda(lambda x: lambda_maccs(x))(ip)
        if MACCS_DROPOUT > 0:
            mx = m_dropout(mx)
        features.append(mx)

    if TARGET_ENABLE:
        t_dropout = Dropout(TARGET_DROPOUT)

        # forward
        tx = Lambda(lambda x: lambda_target(x))(ip)
        if MACCS_DROPOUT > 0:
            tx = t_dropout(tx)
        features.append(tx)

    j_concat = Concatenate()
    j_dropout = Dropout(HIDDEN_DROPOUT)
    j_hidden1 = Dense(HIDDEN_DIMS,
                      kernel_regularizer=l2(HIDDEN_L2),
                      kernel_initializer=HIDDEN_KERNEL_INIT)
    j_bn1 = BatchNormalization()
    j_act1 = ReLU()
    j_cls = Dense(1, activation='sigmoid')

    # Forward
    jx = j_concat(features)
    if HIDDEN_DIMS > 0:
        jx = j_hidden1(jx)
        jx = j_bn1(jx)
        jx = j_act1(jx)

    y = j_cls(jx)

    model = Model(inputs=ip, outputs=y)
    model.compile(optimizer=OPTIMIZER,
                  loss='binary_crossentropy',
                  metrics=['acc'],
                  loss_weights=[1e5])

    if SUMMARY:
        model.summary()

    return model

def build_sklearn():
    return KerasClassifier(build_fn=build_keras,
                           epochs=EPOCHS,
                           batch_size=BATCH_SIZE,
                           verbose=1)

def build_adaboost():
    sklearn_model = build_sklearn()
    model = AdaBoostClassifier(base_estimator=sklearn_model, n_estimators=N_ESTIMATORS)

    return model

def idgc20(y_true, y_pred, cname, dname, submission: bool = False):
    results = {
        'uid': [],
        'y_true': [],
        'y_pred': [],
        'cell': [],
        'drug': []
    }
    if submission:
        del results['y_true']

    for i in range(0, y_pred.shape[0]):
        results['uid'].append(i)
        if not submission:
            results['y_true'].append(y_true[i, 0])
        results['y_pred'].append(y_pred[i, 1])
        results['cell'].append(cname[i])
        results['drug'].append(dname[i])

    df = pd.DataFrame.from_dict(results)
    if submission:
        df['rank'] = 100

    cells_drugs = []

    all_idgc_20 = []
    for cell in df['cell'].unique():

        dfc = df[df['cell'] == cell]
        dfc.sort_values(by=['y_pred'], ascending=False, inplace=True)

        dgc_20 = 0
        for i in range(0, min(20, dfc.shape[0])):
            if submission:
                df.loc[df['uid'] == dfc.iloc[i]['uid'], 'rank'] = i
            if not submission:
                dgc_20 += dfc.iloc[i]['y_true'] / np.log2(i + 2)
        idgc_20 = 0
        for i in range(0, min(20, dfc.shape[0])):
            idgc_20 += 1 / np.log2(i + 2)

        idgc_20 = dgc_20 / idgc_20
        all_idgc_20.append(idgc_20)

    if submission:
        dfs = df[df['rank'] != 100]
        dfs.sort_values(by=['cell', 'rank'], ascending=True, inplace=True)
        dfs.drop(labels=['uid', 'y_pred'], axis=1, inplace=True)
        dfs.to_csv('submission.csv', index=False)

    return np.mean(all_idgc_20)

@plac.annotations(
    cmd=('train, val, rank', 'option', 'c', str)
)
def main(cmd: str = 'train'):
    fix_tf()

    X_train = np.load(os.path.join(TV_PATH, 'X_train.npy'), mmap_mode='r+')
    y_train = np.load(os.path.join(TV_PATH, 'y_train.npy'))
    cname_train = np.load(os.path.join(TV_PATH, 'cname_train.npy'))
    dname_train = np.load(os.path.join(TV_PATH, 'dname_train.npy'))

    X_val = np.load(os.path.join(TV_PATH, 'X_val.npy'), mmap_mode='r+')
    y_val = np.load(os.path.join(TV_PATH, 'y_val.npy'))

    cname_val = np.load(os.path.join(TV_PATH, 'cname_val.npy'))
    dname_val = np.load(os.path.join(TV_PATH, 'dname_val.npy'))

    if cmd == 'train':
        model = build_adaboost()

        model.fit(X_train, y_train)
        y_pred = np.array(model.predict_proba(X_val))

        idgc_20 = idgc20(y_val, y_pred, cname_val, dname_val)

        print("IDGC_20: %f" % idgc_20)

        print("Estimator Weights")
        print(model.estimator_weights_)

        for n, est in enumerate(model.estimators_):
            print("Saving estimator %d..." % n)
            est.model.save_weights('weights/%s_%d.h5' % (SESSION, n))

        model.estimators_ = []
        open('weights/%s.pkl' % SESSION, 'wb').write(pickle.dumps(model))

    elif cmd == 'val':
        model = pickle.loads(open('weights/%s.pkl' % SESSION, 'rb').read())

        for n in range(0, N_ESTIMATORS):
            m_n = build_sklearn()
            m_n.model = build_keras()
            m_n.model.load_weights('weights/%s_%d.h5' % (SESSION, n))
            model.estimators_.append(m_n)

        y_pred = np.array(model.predict_proba(X_val))

        idgc_20 = idgc20(y_val, y_pred, cname_val, dname_val)
        print("IDGC_20: %f" % idgc_20)

    elif cmd == 'rank':
        model = pickle.loads(open('weights/%s.pkl' % SESSION, 'rb').read())

        X_test = np.load(os.path.join(TV_PATH, 'X_test.npy'), mmap_mode='r+')
        cname_test = np.load(os.path.join(TV_PATH, 'cname_test.npy'))
        dname_test = np.load(os.path.join(TV_PATH, 'dname_test.npy'))

        for n in range(0, N_ESTIMATORS):
            m_n = build_sklearn()
            m_n.model = build_keras()
            m_n.model.load_weights('weights/%s_%d.h5' % (SESSION, n))
            model.estimators_.append(m_n)

        y_pred = np.array(model.predict_proba(X_test))

        idgc_20 = idgc20(None, y_pred, cname_test, dname_test, submission=True)

if __name__ == '__main__':
    plac.call(main)


from django import forms
from django.forms import ValidationError

from pgweb.core.models import Organisation
from .models import Event

class EventForm(forms.ModelForm):
    form_intro = 'Before submitting an event, please read the current policy for News and Events'

    toggle_fields = [
        {
            'name': 'isonline',
            'invert': True,
            'fields': ['city', 'state', 'country', ]
        },
    ]

    def __init__(self, *args, **kwargs):
        super(EventForm, self).__init__(*args, **kwargs)
        self.fields['startdate'].help_text = 'Use format YYYY-MM-DD'
        self.fields['enddate'].help_text = 'Use format YYYY-MM-DD'

    def filter_by_user(self, user):
        self.fields['org'].queryset = Organisation.objects.filter(managers=user, approved=True)

    def clean(self):
        cleaned_data = super(EventForm, self).clean()
        if not cleaned_data.get('isonline'):
            # Non online events require city and country
            # (we don't require state, since many countries have no such thing)
            if not cleaned_data.get('city'):
                self._errors['city'] = self.error_class(['City must be specified for non-online events'])
                del cleaned_data['city']
            if not cleaned_data.get('country'):
                self._errors['country'] = self.error_class(['Country must be specified for non-online events'])
                del cleaned_data['country']
        return cleaned_data

    def clean_startdate(self):
        if self.instance.pk and self.instance.approved:
            if self.cleaned_data['startdate'] != self.instance.startdate:
                raise ValidationError("You cannot change the dates on events that have been approved")
        return self.cleaned_data['startdate']

    def clean_enddate(self):
        if self.instance.pk and self.instance.approved:
            if self.cleaned_data['enddate'] != self.instance.enddate:
                raise ValidationError("You cannot change the dates on events that have been approved")
        if 'startdate' in self.cleaned_data and self.cleaned_data['enddate'] < self.cleaned_data['startdate']:
            raise ValidationError("End date cannot be before start date!")
        return self.cleaned_data['enddate']

    class Meta:
        model = Event
        exclude = ('submitter', 'approved', 'description_for_badged')


import urllib.request
import json
import dml
import prov.model
import datetime
import uuid

class trans_school(dml.Algorithm):
    contributor = 'kobesay'
    reads = ['kobesay.publicschool', 'kobesay.nonpublicschool']
    writes = ['kobesay.regionschool', 'kobesay.regionpublicschool', 'kobesay.regionnonpublicschool']

    @staticmethod
    def execute(trial = False):
        '''Retrieve some data sets (not using the API here for the sake of simplicity).'''
        startTime = datetime.datetime.now()

        # Set up the database connection.
        client = dml.pymongo.MongoClient()
        repo = client.repo
        repo.authenticate('kobesay', 'kobesay')

        repo.dropCollection("regionschool")
        repo.createCollection("regionschool")
        repo.dropCollection("regionpublicschool")
        repo.createCollection("regionpublicschool")
        repo.dropCollection("regionnonpublicschool")
        repo.createCollection("regionnonpublicschool")

        # select zip code and school of public schools
        # count number of public schools for each region
        items_publicschool = {}
        publicschool = repo.kobesay.publicschool.find()
        for x in publicschool:
            zipcode = x['fields']['zipcode'].split('-')[0]
            items_publicschool[zipcode] = items_publicschool.get(zipcode, 0) + 1
        r = [{'zipcode': zipcode, 'num': items_publicschool[zipcode]} for zipcode in items_publicschool]
        repo['kobesay.regionpublicschool'].insert_many(r)

        # select zip code and school of nonpublic schools
        # count number of nonpublic schools for each region
        items_nonpublicschool = {}
        nonpublicschool = repo.kobesay.nonpublicschool.find()
        for x in nonpublicschool:
            zipcode = x['properties']['ZIP'].split('-')[0]
            items_nonpublicschool[zipcode] = items_nonpublicschool.get(zipcode, 0) + 1
        r = [{'zipcode': zipcode, 'num': items_nonpublicschool[zipcode]} for zipcode in items_nonpublicschool]
        repo['kobesay.regionnonpublicschool'].insert_many(r)

        # merge result of public school and private school
        # count number of all public/private schools for each region
        items = {}
        for zipcode in items_publicschool:
            items[zipcode] = items.get(zipcode, 0) + items_publicschool[zipcode]
        for zipcode in items_nonpublicschool:
            items[zipcode] = items.get(zipcode, 0) + items_nonpublicschool[zipcode]
        r = [{'zipcode': zipcode, 'num': items[zipcode]} for zipcode in items]
        repo['kobesay.regionschool'].insert_many(r)

        repo.logout()

        endTime = datetime.datetime.now()

        return {"start":startTime, "end":endTime}

    @staticmethod
    def provenance(doc = prov.model.ProvDocument(), startTime = None, endTime = None):
        '''
            Create the provenance document describing everything happening
            in this script. Each run of the script will generate a new
            document describing that invocation event.
            '''

        # Set up the database connection.
        client = dml.pymongo.MongoClient()
        repo = client.repo
        repo.authenticate('kobesay', 'kobesay')
        doc.add_namespace('alg', 'http://datamechanics.io/algorithm/') # The scripts are in <folder>#<filename> format.
        doc.add_namespace('dat', 'http://datamechanics.io/data/') # The data sets are in <user>#<collection> format.
        doc.add_namespace('ont', 'http://datamechanics.io/ontology#') # 'Extension', 'DataResource', 'DataSet', 'Retrieval', 'Query', or 'Computation'.
        doc.add_namespace('log', 'http://datamechanics.io/log/') # The event log.
        doc.add_namespace('bdp', 'https://data.cityofboston.gov/resource/') # City of Boston Data Portal
        doc.add_namespace('bwod', 'https://boston.opendatasoft.com/explore/dataset/') # Boston Wicked Open Data
        doc.add_namespace('bod', 'http://bostonopendata.boston.opendata.arcgis.com/datasets/') # BostonMaps: Open Data

        this_script = doc.agent('alg:kobesay#trans_school', {prov.model.PROV_TYPE:prov.model.PROV['SoftwareAgent'], 'ont:Extension':'py'})
        publicschool = doc.entity('dat:kobesay#publicschool', {'prov:label':'public school', prov.model.PROV_TYPE:'ont:DataSet'})
        nonpublicschool = doc.entity('dat:kobesay#nonpublicschool', {'prov:label':'non public school', prov.model.PROV_TYPE:'ont:DataSet'})
        get_regionschool = doc.activity('log:uuid'+str(uuid.uuid4()), startTime, endTime, {'prov:label':'get region school'})
        get_regionpublicschool = doc.activity('log:uuid'+str(uuid.uuid4()), startTime, endTime, {'prov:label':'get region public school'})
        get_regionnonpublicschool = doc.activity('log:uuid'+str(uuid.uuid4()), startTime, endTime, {'prov:label':'get region non public school'})
        doc.wasAssociatedWith(get_regionschool, this_script)
        doc.wasAssociatedWith(get_regionpublicschool, this_script)
        doc.wasAssociatedWith(get_regionnonpublicschool, this_script)

        regionschool = doc.entity('dat:kobesay#regionschool', {prov.model.PROV_LABEL:'region school', prov.model.PROV_TYPE:'ont:DataSet'})
        doc.wasAttributedTo(regionschool, this_script)
        doc.wasGeneratedBy(regionschool, get_regionschool, endTime)
        doc.wasDerivedFrom(regionschool, publicschool, get_regionschool, get_regionschool, get_regionschool)
        doc.wasDerivedFrom(regionschool, nonpublicschool, get_regionschool, get_regionschool, get_regionschool)

        regionpublicschool = doc.entity('dat:kobesay#regionschool', {prov.model.PROV_LABEL:'region public school', prov.model.PROV_TYPE:'ont:DataSet'})
        doc.wasAttributedTo(regionpublicschool, this_script)
        doc.wasGeneratedBy(regionpublicschool, get_regionpublicschool, endTime)
        doc.wasDerivedFrom(regionpublicschool, publicschool, get_regionpublicschool, get_regionpublicschool, get_regionpublicschool)

        regionnonpublicschool = doc.entity('dat:kobesay#regionnonpublicschool', {prov.model.PROV_LABEL:'region non public school', prov.model.PROV_TYPE:'ont:DataSet'})
        doc.wasAttributedTo(regionnonpublicschool, this_script)
        doc.wasGeneratedBy(regionnonpublicschool, get_regionnonpublicschool, endTime)
        doc.wasDerivedFrom(regionnonpublicschool, nonpublicschool, get_regionnonpublicschool, get_regionnonpublicschool, get_regionnonpublicschool)

        #repo.record(doc.serialize())
        repo.logout()

        return doc
"""
# TEST
trans_school.execute()
doc = trans_school.provenance()
print(doc.get_provn())
print(json.dumps(json.loads(doc.serialize()), indent=4))
"""
## eof


"""
Handling mouse events in pygame

Drag and drop an image
"""

import pygame

BLACK = 0, 0, 0
WHITE = 255, 255, 255
RED = 255, 0, 0
GREEN = 0, 255, 0
BLUE = 0, 0, 255

pygame.init()
canvas = pygame.display.set_mode((500, 600))

people = pygame.image.load('assets/people.png')

# Start with image at 0, 0
people_rect = people.get_rect()

# Are we dragging the image?
dragging = False

done = False
while not done:
    # Blank canvas
    canvas.fill(BLACK)

    mouse_x, mouse_y = pygame.mouse.get_pos()
    for event in pygame.event.get():

        # If mouse button pressed,
        if event.type == pygame.MOUSEBUTTONDOWN:
            # If the mouse pointer is on top
            # of the image
            if people_rect.collidepoint(mouse_x, mouse_y):

                # Start dragging
                dragging = True

        # If the mouse button was released, stop dragging
        elif event.type == pygame.MOUSEBUTTONUP:
            dragging = False

        # Exit when 'q' is pressed
        elif event.type == pygame.KEYDOWN and event.key == pygame.K_q:
            done = True

    # When dragging, move the image to the current
    # mouse position
    if dragging:
        people_rect.left = mouse_x
        people_rect.top = mouse_y

    # Show the image at the (new) location
    canvas.blit(people, people_rect)

    pygame.display.flip()


"""
Problem Set 3
"""

def pretty_print_fwd(fwd_struct):
    """
    Arma cuadro para visualizar una estructura forward.
    Parámetros:
        fwd_struct : Estructura de tasas forward
    """
    for t, fwds in fwd_struct.items():
        fwds_txt = ', '.join(f'{tt:3}: {r:6.2%}' for tt, r in fwds.items())
        print(f'{t:3}: {{', fwds_txt, '}')

if __name__ == '__main__':
    import json

    data = json.loads(open('C:\\Users\\FedericoMenendez\\Desktop\\PYTHON\\FinanzasPython\\data\\c4_mkt_data.json').read())


from factsumm import FactSumm
import jsonlines

factsumm = FactSumm()

with jsonlines.open('/home/tiezheng/workspace/FidSum/data/qmsum/processed_data_no_sent_split/test.jsonl') as F:
    article = []
    target = []
    for item in F:
        article.append(item['src'])
        target.append(item['tgt'])

with open('src.txt', 'w') as F:
    F.writelines([item+'\n' for item in article])

with open('tgt.txt', 'w') as F:
    F.writelines([item+'\n' for item in target])

# with open('/home/tiezheng/workspace/FidSum/data/qmsum/test_results/12_512_no_KA_in_generation_penalty1_maxlen256.results') as F:

#     summary = F.readlines()
#     summary = [item.strip('\n') for item in summary]

# # factsumm(article[0], summary[0], device="cuda")

# article = "Lionel Andrés Messi (born 24 June 1987) is an Argentine professional footballer who plays as a forward and captains both Spanish club Barcelona and the Argentina national team. Often considered as the best player in the world and widely regarded as one of the greatest players of all time, Messi has won a record six Ballon d'Or awards, a record six European Golden Shoes, and in 2020 was named to the Ballon d'Or Dream Team."
# summary = "Lionel Andrés Messi (born 24 Aug 1997) is an Spanish professional footballer who plays as a forward and captains both Spanish club Barcelona and the Spanish national team."
# factsumm(article, summary, verbose=True, device="gpu")


import ssl
from telnetlib import *
from telnetlib.option import TelnetOption

class TelnetOptionAuthentication( TelnetOption ):
    """
        RFC 2941, Telnet Authentication Option
    """
    AUTH_TYPE_NULL          = chr( 0 )
    AUTH_TYPE_KERBEROS_V4   = chr( 1 )
    AUTH_TYPE_KERBEROS_V5   = chr( 2 )
    AUTH_TYPE_SPX           = chr( 3 )
    AUTH_TYPE_MINK          = chr( 4 )
    AUTH_TYPE_SRP           = chr( 5 )
    AUTH_TYPE_RSA           = chr( 6 )
    AUTH_TYPE_SSL           = chr( 7 )
    AUTH_TYPE_unassigned1   = chr( 8 )
    AUTH_TYPE_unassigned2   = chr( 9 )
    AUTH_TYPE_LOKI          = chr( 10 )
    AUTH_TYPE_SSA           = chr( 11 )
    AUTH_TYPE_KEA_SJ        = chr( 12 )
    AUTH_TYPE_KEA_SJ_INTEG  = chr( 13 )
    AUTH_TYPE_DSS           = chr( 14 )
    AUTH_TYPE_NTLM          = chr( 15 )

    AUTHENTICATION_IS       = chr( 0 )
    AUTHENTICATION_SEND     = chr( 1 )
    AUTHENTICATION_REPLY    = chr( 2 )
    AUTHENTICATION_NAME     = chr( 3 )

    def Do( self, telnet ):
        self.log.debug( "IAC WILL AUTHENTICATION" )
        telnet._sock.sendall( IAC + WILL + AUTHENTICATION )
        return
    # end def

    def Execute( self, telnet, sbdataq ):
        self.log.debug( "enter TelnetOptionAuthentication.Execute()" )
        self.log.info( "sbdataq = [%s]" % ( sbdataq.encode( 'hex' ) ) )
        if sbdataq[ 0 ] == AUTHENTICATION:
            action = sbdataq[ 1 ]
            if action == self.AUTHENTICATION_SEND:
                if sbdataq[ 2 ] == self.AUTH_TYPE_SSL:
                    self.log.info( 'AUTHENTICATION.SEND = %d SSL' % ( ord( action ) ) )
                    # <= "ff:fa:25:00:07:00:01:ff:f0"
                    #   IAC SB
                    #       AUTHENTICATION SEND 00 01
                    #   IAC SE
                    self.log.info( 'AUTHENTICATION.IS SSL %02X %02X' % ( 0, 1 ) )
                    self._SendIacSbSe( telnet._sock, AUTHENTICATION + self.AUTHENTICATION_IS + self.AUTH_TYPE_SSL + chr( 0 ) + chr( 1 ) )
                # end if
            elif action == self.AUTHENTICATION_REPLY:
                if sbdataq[ 2 ] == self.AUTH_TYPE_SSL:
                    # => "ff:fa:25:02:07:00:02:ff:f0""
                    #   IAC SB
                    #       AUTHENTICATION REPLY SSL 00 02
                    #   IAC SE
                    self.log.info( 'AUTHENTICATION.REPLY = %d SSL' % ( ord( action ) ) )
                    """
                        Now this start the SSL/TLS connection
                    """
                    sslsock = ssl.wrap_socket( telnet._sock,
                                ciphers="HIGH:-aNULL:-eNULL:-PSK:RC4-SHA:RC4-MD5",
                                # ssl_version=ssl.PROTOCOL_TLSv1,
                                ssl_version=ssl.PROTOCOL_SSLv23 | ssl.PROTOCOL_TLSv1,
                                cert_reqs=ssl.CERT_NONE,
                                ca_certs='/etc/ssl/certs/ca-certificates.crt'
                                )
                    # Start the negotiate
                    sslsock.getpeercert()
                    # Swap the sockets on the TELNET class
                    sock            = telnet._sock
                    telnet._sock    = sslsock
                    telnet._sockssl = sock
                # end if
            else:
                self.log.info( 'AUTHENTICATION.(%d) unsupported' % ( ord( action ) ) )
            # end if
        else:
            # IAC SB <option-bytes> IAC SE
            self.log.info( "IAC SB [%s] IAC SE is unsupported" % ( sbdataq.encode( 'hex' ) ) )
            pass
        # end if
        self.log.debug( "leave TelnetOptionAuthentication.Execute()" )
        return
    # end def
# end class


import abc
from typing import TYPE_CHECKING

from ml_deeco.estimators import Estimate

if TYPE_CHECKING:
    from ml_deeco.simulation import Experiment

class ComponentMeta(abc.ABCMeta):
    """
    Metaclass for Component. Uses a counter to automatically generate the component ID.
    """

    def __new__(mcs, name, bases, namespace):
        namespace['_count'] = 0  # initialize the counter
        return super().__new__(mcs, name, bases, namespace)

class Component(metaclass=ComponentMeta):
    """
    Base class for all components.

    Attributes
    ----------
    id : str
        Identifier of the component. Generated automatically.
    """
    id: str
    _count = 0  # Number of components of each type

    def __init__(self):
        # generate the ID
        cls = type(self)
        cls._count += 1
        self.id = "%s_%d" % (cls.__name__, cls._count)

    def actuate(self):
        """
        Behavior of the component which is executed once per time step. Should be developed by the framework user.
        """
        pass

    def collectEstimatesData(self):
        """
        Collects data for Estimates. This is called from the simulation after a step is performed.
        """
        estimates = [fld for (fldName, fld) in type(self).__dict__.items()
                     if not fldName.startswith('__') and isinstance(fld, Estimate)]
        for estimate in estimates:
            estimate.collectInputs(self)
            estimate.collectTargets(self)

    @classmethod
    def initEstimates(cls, experiment: 'Experiment'):
        """Assigns the experiment to the estimates. This is necessary when the estimator was specified as a string in
        the 'using' method."""
        estimates = [fld for (fldName, fld) in cls.__dict__.items()
                     if not fldName.startswith('__') and isinstance(fld, Estimate)]
        for estimate in estimates:
            estimate.linkExperiment(experiment)

    def __repr__(self) -> str:
        return self.id


import torch.nn as nn
import torch

resl_to_ch = {
    4    : (512, 512),
    8    : (512, 512),
    16   : (512, 512),
    32   : (512, 512),
    64   : (512, 256),
    128  : (256, 128),
    256  : (128, 64),
    512  : (64, 32),
    1024 : (32, 16),
}

resl_to_batch = {
    4    : 512,
    8    : 512,
    16   : 512,
    32   : 256,
    64   : 128,
    128  : 64,
    256  : 32,
    512  : 16,
    1024 : 12
}

resl_to_lr = {
    4    : 0.001,
    8    : 0.001,
    16   : 0.001,
    32   : 0.001,
    64   : 0.0015,
    128  : 0.0015,
    256  : 0.002,
    512  : 0.003,
    1024 : 0.003
}

class EqualizedLR(nn.Module):
    def __init__(self, layer):
        super().__init__()

        nn.init.kaiming_normal_(layer.weight, nonlinearity="relu")

        layer.bias.data.fill_(0)

        self.wscale = layer.weight.data.detach().pow(2.).mean().sqrt()
        layer.weight.data /= self.wscale

        self.layer = layer

    def forward(self, x):
        return self.layer(x * self.wscale)

class PixelWiseNorm(nn.Module):
    def __init__(self):
        super().__init__()

    def forward(self, x):
        x_square_mean = x.pow(2).mean(dim=1, keepdim=True)
        denom = torch.rsqrt(x_square_mean + 1e-8)
        return x * denom

class ToRGBLayer(nn.Module):
    def __init__(self, resl, rgb_channel):
        super().__init__()
        _, in_c  = resl_to_ch[resl]

        self.conv = nn.Sequential(
            EqualizedLR(nn.Conv2d(in_c, rgb_channel, 1)),
            nn.Tanh()
        )

    def forward(self, x):
        return self.conv(x)

class ReslBlock(nn.Module):
    def __init__(self, resl):
        super().__init__()
        in_c, out_c  = resl_to_ch[resl]

        self.conv = nn.Sequential(
            EqualizedLR(nn.Conv2d(in_c, out_c, 3, 1, 1)),
            PixelWiseNorm(),
            nn.LeakyReLU(negative_slope=0.2, inplace=True),
            EqualizedLR(nn.Conv2d(out_c, out_c, 3, 1, 1)),
            PixelWiseNorm(),
            nn.LeakyReLU(negative_slope=0.2, inplace=True),
        )

    def forward(self, x):
        up   = F.interpolate(x, scale_factor=2)
        conv = self.conv(up)
        return conv

class G(nn.Module):
    def __init__(self, resl=4, rgb_channel=3):
        super().__init__()
        self.resl = resl
        self.rgb_channel = rgb_channel

        in_c, out_c = resl_to_ch[resl]

        self.resl_blocks = nn.Sequential(
            # Original Repo using "tf.nn.conv2d_transpose"
            EqualizedLR(nn.ConvTranspose2d(in_c, out_c, 4)),
            # nn.Upsample(size=(4, 4)),
            # EqualizedLR(nn.Conv2d(in_c, out_c, 1, 1, 0)),
            PixelWiseNorm(),
            nn.LeakyReLU(negative_slope=0.2, inplace=True),
            EqualizedLR(nn.Conv2d(out_c, out_c, 3, 1, 1)),
            PixelWiseNorm(),
            nn.LeakyReLU(negative_slope=0.2, inplace=True),
        )

        self.rgb_l = None
        self.rgb_h = ToRGBLayer(self.resl, self.rgb_channel)
        self.alpha = 0

    def forward(self, x, phase):
        if phase == "transition":
            return self.transition_forward(x)
        elif phase == "stabilization":
            return self.stabilization_forward(x)

    def grow_network(self):
        self.resl *= 2
        self.resl_blocks = nn.Sequential(*self.resl_blocks, ReslBlock(self.resl))
        self.rgb_l = self.rgb_h
        self.rgb_h = ToRGBLayer(self.resl, self.rgb_channel)
        self.alpha = 0

    def transition_forward(self, x):
        x = self.resl_blocks[:-1](x)

        # experiment candidate : apply rgb_l first and succeeding interpolate
        # low resolution path
        x_up = F.interpolate(x, scale_factor=2)
        rgb_l = self.rgb_l(x_up)

        # high resolution path
        x = self.resl_blocks[-1](x)
        rgb_h = self.rgb_h(x)

        return self.alpha * (rgb_h - rgb_l) + rgb_l

    def stabilization_forward(self, x):
        x = self.resl_blocks(x)
        rgb_h = self.rgb_h(x)
        return rgb_h

    def update_alpha(self, delta):
        self.alpha += delta
        self.alpha = min(1, self.alpha)

if __name__ == '__main__':
    gg = G()
    print(gg)
    print('-'*100)

#    gg.grow_network()
#    print(gg)
#    print('-'*100)
#
#    gg.grow_network()
#    print(gg)
#    print('-'*100)


"""
Created on 7 Apr 2018

@author: john.dwan

ELEVATION PROFILE APP GENERATOR
ideagora geomatics-2018
http://geodose.com
"""

import json
from math import asin, cos, radians, sin, sqrt
import urllib.request

import matplotlib.pyplot as plt
from shapely.geometry import LineString

from siteline import config

def haversine(lat1, lon1, lat2, lon2):
	"""
	Given the coordinates of two points as floats

	Returns the distance between these two point in km
	"""
	lat1_rad = radians(lat1)
	lat2_rad = radians(lat2)
	lon1_rad = radians(lon1)
	lon2_rad = radians(lon2)
	delta_lat = lat2_rad - lat1_rad
	delta_lon = lon2_rad - lon1_rad
	a = sqrt((sin(delta_lat / 2)) ** 2 + cos(lat1_rad) * cos(lat2_rad) * (sin(delta_lon / 2)) ** 2)
	d = 2 * 6371000 * asin(a)
	return d

def plot_profile(
		a_end_name, a_end_lat, a_end_lon, b_end_name, b_end_lat, b_end_lon, a_end_agl=10, b_end_agl=20, num_points=1000):
	"""
	Takes a customer site (a_end_name=str), it's coordinates (a_end_lat=Float, a_end_lon=Float) and a
	Highsite (b_end_name=str) and it's coordinates (b_end_lat=Float, b_end_lon=Float)
	and optionally number of points along a line (num_points=Integer) which is max 1000 on open-elevation.com API,
	Note: no limit if running open-elevation locally.

	Saves a elevation profile between these two points as a .png
	"""

	# NUMBER OF POINTS
	interval_lat = (b_end_lat - a_end_lat) / num_points  # interval for latitude
	interval_lon = (b_end_lon - a_end_lon) / num_points  # interval for longitude

	# LATITUDE AND LONGITUDE LIST
	lat_list = [a_end_lat]
	lon_list = [a_end_lon]

	# GENERATING POINTS
	for i in range(num_points):
		lat_step = a_end_lat + interval_lat
		lon_step = a_end_lon + interval_lon
		a_end_lon = lon_step
		a_end_lat = lat_step
		lat_list.append(lat_step)
		lon_list.append(lon_step)

	# DISTANCE CALCULATION
	d_list = []
	for j in range(len(lat_list)):
		lat_p = lat_list[j]
		lon_p = lon_list[j]
		dp = haversine(a_end_lat, a_end_lon, lat_p, lon_p) / 1000  # km
		d_list.append(dp)
	d_list_rev = d_list[::-1]  # reverse list

	# CONSTRUCT JSON
	location = {}
	d_ar = [{}] * len(lat_list)
	for i in range(len(lat_list)):
		d_ar[i] = {"latitude": lat_list[i], "longitude": lon_list[i]}
	location['locations'] = d_ar
	json_data = json.dumps(location, skipkeys=int).encode('utf8')

	# SEND REQUEST
	response = urllib.request.Request(config.URL, json_data, headers={'Content-Type': 'application/json'})
	fp = urllib.request.urlopen(response)

	# RESPONSE PROCESSING
	res_byte = fp.read()
	res_str = res_byte.decode("utf8")
	js_str = json.loads(res_str)
	fp.close()

	# GETTING ELEVATION
	response_len = len(js_str['results'])
	elev_list = []
	for j in range(response_len):
		elev_list.append(js_str['results'][j]['elevation'])

	# BASIC STAT INFORMATION
	# mean_elev = round((sum(elev_list) / len(elev_list)), 3)
	# min_elev = min(elev_list)
	# max_elev = max(elev_list)
	distance = d_list_rev[-1]

	# LoS
	a_list = []
	for i in range(0, len(d_list_rev) - 1):
		a_tuple = (d_list_rev[i], elev_list[i])
		a_list.append(a_tuple)
	terrain = LineString(a_list)
	los_path = LineString(
		[(d_list_rev[0], elev_list[0] + a_end_agl), (d_list_rev[-1], elev_list[-1] + b_end_agl)])

	path_colour = '--g'
	print(terrain.intersection(los_path))
	if terrain.intersection(los_path):
		print('No LoS')
		path_colour = '--r'

	# PLOT ELEVATION PROFILE
	# with context('fivethirtyeight'):
	base_reg = 0
	plt.figure(figsize=(10, 4))
	plt.plot(d_list_rev, elev_list)
	plt.plot(
		[d_list_rev[0],
		d_list_rev[-1]],
		[elev_list[0] + a_end_agl,
		elev_list[-1] + b_end_agl],
		path_colour,
		label='LoS 10m AGL customer site to 20m highsite'
	)
	plt.fill_between(d_list_rev, elev_list, base_reg, alpha=0.1)
	plt.text(d_list_rev[0], elev_list[0], a_end_name)
	plt.text(d_list_rev[-1], elev_list[-1], b_end_name)
	plt.xlabel('Distance(km)    ' + str(round(distance)) + 'km')
	plt.ylabel('Elevation(m)    ' + str(elev_list[-1] - elev_list[0]) + 'm')
	plt.grid()
	plt.legend(fontsize='small')
	fname = a_end_name + '__to__' + b_end_name + '.png'
	plt.savefig(config.OUT_PATH.joinpath(fname))
	plt.close()


"""Common methods and classes used for mesh client"""
from collections import namedtuple
import os
import json
import boto3

REGION_NAME = os.environ.get("AWS_REGION", "eu-west-2")

class SingletonCheckFailure(Exception):
    """Singleton check failed"""

    def __init__(self, msg=None):
        super().__init__()
        self.msg = msg

class AwsFailedToPerformError(Exception):
    """Errors raised by AWS functions"""

    def __init__(self, msg=None):
        super().__init__()
        self.msg = msg

class MeshCommon:
    """Common"""

    MIB = 1024 * 1024
    DEFAULT_CHUNK_SIZE = 20 * MIB

    @staticmethod
    def singleton_check(mailbox, my_step_function_name):
        """Find out whether there is another step function running for my mailbox"""
        sfn_client = boto3.client("stepfunctions", region_name="eu-west-2")
        response = sfn_client.list_state_machines()
        # Get my step function arn
        my_step_function_arn = None
        for step_function in response.get("stateMachines", []):
            if step_function.get("name", "") == my_step_function_name:
                my_step_function_arn = step_function.get("stateMachineArn", None)

        # TODO add this check to tests
        if not my_step_function_arn:
            raise SingletonCheckFailure(
                "No executing step function arn for "
                + f"step_function={my_step_function_name}"
            )

        response = sfn_client.list_executions(
            stateMachineArn=my_step_function_arn,
            statusFilter="RUNNING",
        )
        currently_running_step_funcs = []
        for execution in response["executions"]:
            currently_running_step_funcs.append(execution["executionArn"])

        exec_count = 0
        for execution_arn in currently_running_step_funcs:
            response = sfn_client.describe_execution(executionArn=execution_arn)
            step_function_input = json.loads(response.get("input", "{}"))
            input_mailbox = step_function_input.get("mailbox", None)
            if input_mailbox == mailbox:
                exec_count = exec_count + 1
            if exec_count > 1:
                raise SingletonCheckFailure("Process already running for this mailbox")

        return True

    @staticmethod
    def convert_params_to_dict(params):
        """Convert paramater dict to key:value dict"""
        new_dict = {}
        for entry in params:
            name = entry.get("Name", None)
            if name:
                var_name = os.path.basename(name)
                new_dict[var_name] = entry.get("Value", None)
        return new_dict

    @staticmethod
    def return_failure(log_object, status, logpoint, mailbox, message=""):
        """Return a failure response with retry"""
        log_object.write_log(logpoint, None, {"mailbox": mailbox, "error": message})
        return {
            "statusCode": status,
            "headers": {
                "Content-Type": "application/json",
                "Retry-After": 18000,
            },
            "body": {
                "internal_id": log_object.internal_id,
                "error": message,
            },
        }

    @staticmethod
    def get_params(path, recursive=False, decryption=True):
        """
        Get parameters from SSM and secrets manager
        """
        ssm_client = boto3.client("ssm", region_name=REGION_NAME)
        params_result = ssm_client.get_parameters_by_path(
            Path=path,
            Recursive=recursive,
            WithDecryption=decryption,
        )
        params = params_result.get("Parameters", {})
        new_params_dict = {}
        for entry in params:
            name = entry.get("Name", None)
            if name:
                var_name = os.path.basename(name)
                new_params_dict[var_name] = entry.get("Value", None)
        if os.environ.get("use_secrets_manager") == "true":
            secrets_client = boto3.client("secretsmanager", region_name=REGION_NAME)
            all_secrets_dict = secrets_client.list_secrets()
            all_secrets_list = all_secrets_dict["SecretList"]
            for secret in all_secrets_list:
                name = secret["Name"]
                if name.startswith(path):
                    secret_value = secrets_client.get_secret_value(SecretId=name)[
                        "SecretString"
                    ]
                    var_name = os.path.basename(name)
                    new_params_dict[var_name] = secret_value
        return new_params_dict

# Named tuple for holding Mesh Message info
MeshMessage = namedtuple(
    "MeshMessage",
    [
        "filename",
        "body",
        "src_mailbox",
        "dest_mailbox",
        "workflow_id",
        "message_id",
    ],
)


import numpy as np
import cv2
import time
from framecounter import FrameCounter

fcount = FrameCounter()

cv2.startWindowThread()
# cap = cv2.VideoCapture('walking.mp4')
cap = cv2.VideoCapture(0)

i = 0
while(True):
    i+=1
    ret, frame = cap.read()

    fcount.start()
    frame = cv2.resize(frame,(640,360))
    #frame = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)
    # ret,frame = cv2.threshold(frame,80,255,cv2.THRESH_BINARY)

    cv2.imshow('frame',frame)
    if cv2.waitKey(1) & 0xFF == ord('q'):
       # breaking the loop if the user types q
       # note that the video window must be highlighted!
       break
    fcount.stop()

    if i%10==0:
        print(fcount)

cap.release()
# cv2.destroyAllWindows()
# the following is necessary on the mac,
# maybe not on other platforms:
# cv2.waitKey(1)


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import math
import re
import random
import collections
from scipy import stats

import jieba
import jieba.posseg as pseg
import wordcloud
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD
from sklearn.cluster import KMeans

# ==============导入数据
data = pd.read_csv('tech_bv_info.csv', encoding='utf-8')
print('============data==========')
print(data)
print(data.info())

# ==============数据预处理
# 硬币、收藏、分享
def str_num_trans(x):
    if x=='投币':
        return 0
    if x=='收藏':
        return 0
    if re.search('万',x) is not None:
        return int(float(x.replace('万',''))*10000)
    return int(x)
data['coin'] = data['coin'].map(str_num_trans)
data['collect'] = data['collect'].map(str_num_trans)
data['share'] = data['share'].map(str_num_trans)

# 简介、评论去站内链接、换行
def text_process(x):
    # 去链接
    idx = re.search('<a class((?:.|\n)*)',x)
    if idx is not None:
        idx = idx.span()
        x_a_sub = x[idx[0]:idx[1]]
        a_sub = re.sub('','',x_a_sub.replace('',''))
        x = x.replace(x_a_sub,a_sub)
    # 去
    x = re.sub('<(/*)span>','',x)
    return x
def desc_process(x):
    if pd.isna(x):
        return x
    return text_process(x)
def comment_process(x):
    if pd.isna(x):
        return x
    comment_list = x.split('|$comment-sep$|')
    new_list = []
    for comment in comment_list:
        new_list.append(text_process(comment))
    return '|$comment-sep$|'.join(new_list)
data['describe'] = data['describe'].map(desc_process)
data['comment'] = data['comment'].map(comment_process)

# tag处理，去空格
def tag_process(x):
    tag_list = x.split('|$tag-sep$|')
    new_list = []
    for tag in tag_list:
        new_list.append(tag.replace(' ','').replace('\n',''))
    new_list = list(set(new_list))
    return '|$tag-sep$|'.join(new_list)
data['tag'] = data['tag'].map(tag_process)

# =================数据描述性分析
def count_table(df):
    table = pd.DataFrame()
    for col in df.columns.tolist():
        if df[col].dtype == object:
            continue
        table.loc[col,'max'] = df[col].max()
        table.loc[col,'min'] = df[col].min()
        table.loc[col,'mean'] = round(df[col].mean(),2)
        table.loc[col,'median'] = df[col].median()
    return table
print('============各字段统计量表格==============')
print(count_table(data))

# log变换后，集中在1w-3w
plt.hist(data['play'].map(math.log).to_numpy(),color='#D2B48C')
plt.xlabel('播放量/log')
plt.ylabel('数量')
plt.title('视频播放量分布')
plt.rcParams['font.sans-serif'] = ['SimHei']
plt.show()

# 因为是数码科技专区，男性多可以理解
plt.bar(['男','保密','女'],data['author_sex'].value_counts().tolist(), color=['#BDB76B','#F0E68C','#EEE8AA'])
plt.xlabel('性别')
plt.ylabel('数量')
plt.title('up主性别柱状图')
plt.show()

time_videoNum = data['time'].map(lambda x:x[0:10]).value_counts().sort_index(ascending=True)
plt.bar(x=time_videoNum.index.tolist(), height=time_videoNum.tolist(), color='#FA8072')
plt.xlabel('发布时间')
plt.xticks(time_videoNum.index.tolist()[::15],rotation=70)
plt.ylabel('视频数')
plt.title('视频发布时间的分布')
plt.show()

def if_topic(x):
    if '|$tag-topic$|' in x:
        return 1
    return 0
data['event_join'] = data['tag'].map(if_topic)
plt.boxplot([data.loc[data['event_join']==1,'play'].map(math.log).tolist(),
             data.loc[data['event_join']==0,'play'].map(math.log).tolist()],
            labels=['参加','不参加'],
            flierprops={'markerfacecolor':'#CD5C5C'},
            whiskerprops={'color':'#800000'},
            patch_artist=True,
            boxprops={'facecolor':'#DEB887'},
            medianprops={'color':'#800000'})
plt.xlabel('是否参加创作激励活动')
plt.ylabel('播放量/log')
plt.title('参与活动话题与视频播放量')
plt.show()

print('未参加：{}({:.2f})，参加：{}({:.2f})'.format(data[data['event_join']==0].shape[0],data[data['event_join']==0].shape[0]/data.shape[0],
                                    data[data['event_join']==1].shape[0],data[data['event_join']==1].shape[0]/data.shape[0]))
# t检验
print(
    stats.ttest_ind(data.loc[data['event_join']==0,'play'].map(math.log).to_numpy(), data.loc[data['event_join']==1,'play'].map(math.log).to_numpy())
)

plt.boxplot([data.loc[data['duration']<=60,'play'].map(math.log).tolist(),
             data.loc[(data['duration']>60)&(data['duration']<=300),'play'].map(math.log).tolist(),
             data.loc[(data['duration']>300)&(data['duration']<=600),'play'].map(math.log).tolist(),
             data.loc[(data['duration']>600)&(data['duration']<=900),'play'].map(math.log).tolist(),
             data.loc[(data['duration']>900)&(data['duration']<=1800),'play'].map(math.log).tolist(),
             data.loc[(data['duration']>1800)&(data['duration']<=3600),'play'].map(math.log).tolist(),
             data.loc[data['duration']>3600,'play'].map(math.log).tolist()],
            labels=['1分钟以内','1-5分钟','5-10分钟','10-15分钟','15-30分钟','30-60分钟','1小时以上'],
            flierprops={'markerfacecolor':'#CD5C5C'},
            whiskerprops={'color':'#800000'},
            patch_artist=True,
            boxprops={'facecolor':'#DEB887'},
            medianprops={'color':'#800000'})
plt.xlabel('视频时长')
plt.ylabel('播放量/log')
plt.title('视频时长与视频播放量')
plt.show()

plt.scatter(data['author_fans'].map(lambda x:math.log(x) if x!=0 else 0).tolist(), data['play'].map(math.log).tolist(),
            s=4,c='#BDB76B')
plt.xlabel('粉丝数/log')
plt.ylabel('播放量/log')
plt.title('粉丝量与视频播放量')
plt.show()

with open('my_stop_words.txt', 'r', encoding='utf-8') as f:
    stop_words = f.read().split('\n')
title_list = data['title'].tolist()
title_words = []
for i in title_list:
    for word in jieba.cut(i):
        if word not in stop_words and len(re.findall('[0-9]',word))!=len(word):
            title_words.append(word)
title_word_freq = sorted(collections.Counter(title_words).items(), key=lambda x:x[1], reverse=True)

title_words_wc = ' '.join(title_words)

wc=wordcloud.WordCloud(font_path="C:\Windows\Fonts\simfang.ttf",width=1000, height=800,
                             max_words=100,
                             background_color='white',colormap='summer')
wc.generate_from_text(title_words_wc)

plt.imshow(wc)

plt.bar(['小米','华为','苹果','iPhone','三星','红米','荣耀'],[269,205,191,147,123,95,70],
        color=['#006400','#008000','#008000','#228B22','#32CD32','#00FF00','#00FF7F'])
plt.xlabel('手机品牌')
plt.ylabel('频数')
plt.title('不同手机品牌出现频数')
plt.show()

phone = []
pc = []
gpu = []
others = []
for i in range(data.shape[0]):
    if '手机' in data.loc[i,'title']:
        phone.append(data.loc[i,'play'])
    elif '显卡' in data.loc[i,'title']:
        gpu.append(data.loc[i,'play'])
    elif '笔记本' in data.loc[i,'title']:
        pc.append(data.loc[i,'play'])
    else:
        others.append(data.loc[i,'play'])
plt.bar(height=[np.array(phone).mean(), np.array(pc).mean(), np.array(gpu).mean(), np.array(others).mean()],
            x=['手机','笔记本','显卡','其他'],color=['#4169E1','#6495ED','#B0C4DE','#708090'])
plt.xlabel('视频类型')
plt.ylabel('平均播放量')
plt.show()
# 标题带有手机、笔记本、显卡的视频平均播放量明显较高

# ========================文本分析
hot_topic = []
hot_bv = []
for i in range(data.shape[0]):
    if data.loc[i,'play']>1000000:
        hot_bv.append(data.loc[i,'bv'])
        hot_video = []
        for word in jieba.cut(data.loc[i,'title']):
            if word not in stop_words:
                hot_video.append(word)
        for word in data.loc[i,'tag'].split('|$tag-sep$|'):
            if word not in hot_video:
                hot_video.append(word)
        hot_topic+=hot_video
hot_topic = sorted(collections.Counter(hot_topic).items(), key=lambda x:x[1], reverse=True)
print(hot_topic)

hot_wc = []
for i in range(11):
    topic=['科技猎手','苹果','华为','安卓','创意','DIY','自制','AMD','性能','发布会','三星']
    topic_num=[83,28,15,15,7,7,7,6,6,6,5]
    for k in range(topic_num[i]):
        hot_wc.append(topic[i])
random.shuffle(hot_wc)
hot_wc = ' '.join(hot_wc)
wc=wordcloud.WordCloud(font_path="C:\Windows\Fonts\simfang.ttf",width=500, height=400,
                             max_words=100,
                             background_color='white',colormap='spring')
wc.generate_from_text(hot_wc)

plt.imshow(wc)

def comment_process2(x):
    if pd.isna(x):
        return x
    com_li = x.split('|$comment-sep$|')
    new_li = []
    for com in com_li:
        new_li.append(com.replace('\n','。').replace('|$comment-top$|',''))
    return '。'.join(new_li)
data['comment_processed'] = data['comment'].map(comment_process2)

com_wc = []
for com in data['comment_processed']:
    if not pd.isna(com):
        for word,flag in pseg.cut(com):
            if word not in stop_words and word not in ['…','_',':'] and word not in ['doge','辣','眼睛','藏狐'] and 'n' in flag:
                com_wc.append(word)
comment_word_freq = sorted(collections.Counter(com_wc).items(), key=lambda x:x[1], reverse=True)

comment_wc = ' '.join(com_wc)

wc=wordcloud.WordCloud(font_path="C:\Windows\Fonts\simfang.ttf",width=1000, height=800,
                             max_words=100,
                             background_color='white',colormap='copper')
wc.generate_from_text(comment_wc)

plt.imshow(wc)

com_wc = []
for com in data['comment_processed']:
    if not pd.isna(com):
        for word in jieba.cut(com):
            if word not in stop_words and word not in ['…','_',':']:
                com_wc.append(word)
comment_word_freq = sorted(collections.Counter(com_wc).items(), key=lambda x:x[1], reverse=True)
wf1 = []
wf2 = []
for i in title_word_freq[:50]:
    wf1.append(i[0])
for i in comment_word_freq[:100]:
    wf2.append(i[0])
wf1s = set(wf1)
wf2s = set(wf2)
wf = wf2s-wf1s
com_unique_freq = []
for i in wf:
    if len(re.findall('[0-9]',i))!=len(i) and len(re.findall('[a-zA-Z]',i))!=len(i) and i not in '说好很没会真的人想笑哭做去更下感觉再里后才很多辣眼睛太中脱单藏狐出东西妙一点不错确实真希望建议只能':
        com_unique_freq.append(comment_word_freq[wf2.index(i)])
com_unique_freq = sorted(com_unique_freq, key=lambda x:x[1], reverse=True)

plt.figure(figsize=(10,6))
plt.bar([i[0] for i in com_unique_freq], [i[1] for i in com_unique_freq],
        color=['#BDB76B','#EEE8AA','#F0E68C','#FFFACD','#FFF8DC'])
plt.xlabel('热评关键词')
plt.ylabel('词频')
plt.title('热评')
plt.show()

# ===================聚类推荐系统
def describe_process2(x):
    if pd.isna(x):
        return x
    return x.replace('\n','。')
data['describe_processed'] = data['describe'].map(describe_process2)
def tag_process2(x):
    if pd.isna(x):
        return x
    return x.replace('|$tag-sep$|','。').replace('|$tag-topic$|','')
data['tag_processed'] = data['tag'].map(tag_process2)

document = []
for i in range(data.shape[0]):
    doc = []
    for word in jieba.cut(data.loc[i,'title']):
        if word not in stop_words and word not in ['…','_',':']:
            doc.append(word)
    if not pd.isna(data.loc[i,'tag_processed']):
        for word in jieba.cut(data.loc[i,'tag_processed']):
            if word not in stop_words and word not in ['…','_',':']:
                doc.append(word)
    if not pd.isna(data.loc[i,'describe_processed']):
        for word in jieba.cut(data.loc[i,'describe_processed']):
            if word not in stop_words and word not in ['…','_',':']:
                doc.append(word)
    if not pd.isna(data.loc[i,'comment_processed']):
        for word in jieba.cut(data.loc[i,'comment_processed']):
            if word not in stop_words and word not in ['…','_',':']:
                doc.append(word)
    document.append(' '.join(doc))

tfidf = TfidfVectorizer()
video_mat = tfidf.fit_transform(document)
svd = TruncatedSVD(n_components=240)
video_mat_svd = svd.fit_transform(video_mat)
km = KMeans(n_clusters=100)
video_clu = km.fit_predict(video_mat_svd)
data['clu']=video_clu
print('==================聚类为第一类的视频====================')
print(data.loc[data['clu']==1,['bv','title','play','like','duration']])


from django.conf.urls.defaults import *
from django.contrib.auth.views import login, logout_then_login
from dpro.bilet.views import * #index, doldur, listele, csvdeneme, degistir,

urlpatterns = patterns(
    '',
    (r'^bilet/admin/',   include('django.contrib.admin.urls')),
    (r'^bilet/login/$',  login, {'template_name': 'login.html'}),
    (r'^bilet/logout/$', logout_then_login),
    (r'^bilet/index/$',  index),
    (r'^bilet/biletdoldur/([H, O, D])/(\d{1,})/$', doldur),
    (r'^bilet/biletlistele/(\d{1,})/$',    listele),
#    (r'^bilet/csvdeneme/$', csvdeneme),
    (r'^bilet/degistir/(\d{1,})/$', degistir),
    (r'^bilet/biletsil/(\d{1,})/$', sil),
    (r'^bilet/ara/$', ara),
)


from workers.serializers import *
from workers.models import *
from projects.models import *
from rest_framework import viewsets, permissions, status
from rest_framework.response import Response
from rest_framework.views import APIView
from django.db.models import Q
from django.db.models import Prefetch
import datetime
from teams.serializers import *
from teams.models import *

def excluding_keys(dict, keys):
    return {x: dict[x] for x in dict if x not in keys}

class WorkerViewSet(viewsets.ModelViewSet):
    queryset = Worker.objects.all()
    serializer_class = WorkerSerializer
    permission_classes = [permissions.AllowAny]

class WorkerList(APIView):
    permission_classes = [permissions.AllowAny]

    def post(self, request):

        data = request.data
        worker_data = excluding_keys(data, 'team')
        worker_serializer = WorkerSerializer(data = worker_data)
        if worker_serializer.is_valid():
            worker_serializer.save()

            try:
                created_worker = Worker.objects.get(id = worker_serializer.data['id'])
                worker_team = Team.objects.get(id = data['team'])
                new_membership = TeamMembership(worker = created_worker, team = worker_team)
                new_membership.save()
            except:
                return Response(
                    "Something went wrong",
                    status=status.HTTP_400_BAD_REQUEST)

            return Response({
					'worker_data': worker_serializer.data,
					'team': data['team']
				})

        else:
            return Response(worker_serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class WorkerDetail(APIView):
    permission_classes = [permissions.AllowAny]

    def put(self, request, pk):
        editing_data = excluding_keys(request.data, 'team')
        team_id = request.data['team']

        editing_worker = Worker.objects.get(id = pk)
        worker_serializer = WorkerSerializer(editing_worker, editing_data)
        if worker_serializer.is_valid():
            worker_serializer.save()
            try:
                editing_membership = TeamMembership.objects.filter(worker = pk)[0]
                membership_serializer = TeamMembershipSerializer(editing_membership, {
                    'worker': pk, 'team': team_id})
                if membership_serializer.is_valid():
                    membership_serializer.save()
                    return Response({
                        'worker_data': worker_serializer.data,
                        'team_data': membership_serializer.data
                    })
                else:
                    return Response(membership_serializer.errors, status=status.HTTP_400_BAD_REQUEST)

            except TeamMembership.DoesNotExist:
                return Response(f"Membership from worker {pk} not exist", status=status.HTTP_404_NOT_FOUND)

        else:
            return Response(worker_serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class WeekViewSet(viewsets.ModelViewSet):
    queryset = Week.objects.all()
    serializer_class = WeekSerializer
    permission_classes = [permissions.AllowAny]

class JobTimeViewSet(viewsets.ModelViewSet):
    queryset = JobTime.objects.all()
    serializer_class = JobTimeSerializer
    permission_classes = [permissions.AllowAny]

    def create(self, request, *args, **kwargs):
        data = request.data
        many = isinstance(data, list)
        serializer = self.get_serializer(data=data, many=many)
        serializer.is_valid(raise_exception=True)
        self.perform_create(serializer)
        headers = self.get_success_headers(serializer.data)
        return Response(
            serializer.data,
            status=status.HTTP_201_CREATED,
            headers=headers
        )

MONTH_LIST = [
    'Styczeń',
    'Luty',
    'Marzec',
    'Kwiecień',
    'Maj',
    'Czerwiec',
    'Lipiec',
    'Sierpień',
    'Wrzesień',
    'Październik',
    'Listopad',
    'Grudzień']

class JobTimeAvailabilityDetail(APIView):
    permission_classes = [permissions.AllowAny]

    def get(self, request, pk):
        week_id = pk

        try:
            Week.objects.get(id = week_id)

            response_data = []
            workers = Worker.objects.filter(role = 'W')
            for worker in workers:
                worker_id = worker.id
                job_time = JobTime.objects.filter(
                    Q(worker = worker_id) & Q(week = week_id))

                if len(job_time) > 0:
                    availability = job_time[0].availability
                    worker_team = TeamMembership.objects.filter(worker = 5)[0].team

                    response_data.append({
                        'id': worker.id,
                        'name': f"{worker.firstName} {worker.lastName}",
                        'time': availability,
                        'team': {
                            'id': worker_team.id,
                            'name': worker_team.shortcut
                        }
                    })

            return Response(response_data)

        except Week.DoesNotExist:
            return Response(f"Week {week_id} not in db", status=status.HTTP_404_NOT_FOUND)

    def put(self, request, pk):
        full_data = request.data["data"]
        week_id = request.data["week_id"]

        response_data = []

        for single_data in full_data:
            worker_id = single_data["id"]
            time = single_data["time"]

            job_list = JobTime.objects.filter(Q(worker = worker_id) & Q(week = week_id))
            for single_job in job_list:
                data_to_put = {
                    "worker": single_job.worker.id,
                    "project": single_job.project.id,
                    "week": single_job.week.id,
                    "availability": time
                }
                editing_jobtime = JobTime.objects.get(id = single_job.id)
                jobtime_serializer = JobTimeSerializer(editing_jobtime, data_to_put)
                if jobtime_serializer.is_valid():
                    jobtime_serializer.save()
                    response_data.append(jobtime_serializer.data)
                else:
                    return Response(jobtime_serializer.errors, status=status.HTTP_400_BAD_REQUEST)

        return Response({'data': response_data})

class WorkerProjectsDetail(APIView):
    permission_classes = [permissions.AllowAny]

    def put(self, request, pk):
        full_data = request.data["data"]
        week_id = request.data["week_id"]

        response_data = []

        for single_data in full_data:
            worker_id = single_data["worker_id"]
            projects = single_data["projects"]

            for project in projects:
                try:
                    job_time = JobTime.objects.get(
                        Q(worker = worker_id) & Q(week = week_id) & Q(project = project['id']))
                    editing_jobtime = JobTime.objects.get(id = job_time.id)
                    job_time = {
                        'id': job_time.id,
                        'worker': worker_id,
                        'project': project['id'],
                        'workPart': project['time'],
                    }
                    jobtime_serializer = JobTimeSerializer(editing_jobtime, job_time)
                    if jobtime_serializer.is_valid():
                        jobtime_serializer.save()
                        response_data.append(jobtime_serializer.data)
                    else:
                        return Response(jobtime_serializer.errors, status=status.HTTP_400_BAD_REQUEST)

                except JobTime.DoesNotExist:
                    new_job_time = JobTime(
                        worker = Worker.objects.get(id = worker_id),
                        project = Project.objects.get(id = project['id']),
                        week = Week.objects.get(id = week_id),
                        workPart = project['time'])
                    new_job_time.save()
                    response_data.append({
                        'id': new_job_time.id,
                        'worker': worker_id,
                        'project': project['id'],
                        'workPart': project['time'],
                    })

        return Response({'data': response_data})

class WeekList(APIView):
    permission_classes = [permissions.AllowAny]

    def get(self, request, pk):
        weeks = Week.objects.filter(dateEnd__year=pk)
        new_weeks = []
        new_end = datetime.date(1, 1, 1)
        if len(weeks) == 0:
            new_end = datetime.date(pk, 1, 1)
            while new_end.weekday() != 6:
                new_end += datetime.timedelta(days = 1)
        else:
            last_week = weeks[len(weeks) - 1]
            new_end = last_week.dateEnd + datetime.timedelta(days = 7)

        while new_end.year != pk + 1:
            new_start = new_end - datetime.timedelta(days = 6)
            week_name = 'T' + str(new_end.isocalendar().week)
            new_weeks.append({
                'weekName': week_name,
                'dateStart': new_start,
                'dateEnd': new_end,
                'hours': 40
            })
            new_end += datetime.timedelta(days = 7)

        if len(new_weeks) > 0:
            serializer = WeekSerializer(data = new_weeks, many = True)
            if serializer.is_valid():
                serializer.save()
            else:
                return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

        result = Week.objects.filter(
            (Q(dateStart__year = pk) & ~Q(dateEnd__month = 12)) |
            (Q(dateEnd__year = pk) & ~Q(dateEnd__month = 1)))
        serializer = WeekYearSerializer(result, many=True)

        pre_final = [{'month': i, 'weeks': []} for i in MONTH_LIST]
        for record in serializer.data:
            for i in range(1, 13):
                if i in record['months']:
                    pre_final[i - 1]['weeks'].append(excluding_keys(record, 'months'))

        final = []
        for month in pre_final:
            if len(month['weeks']) != 0:
                final.append(month)

        return Response(final)

class WorkerAvailabilityViewSet(viewsets.ModelViewSet):
    queryset = Worker.objects.all()
    serializer_class = WorkerAvailabilitySerializer
    permission_classes = [permissions.AllowAny]

class WeekYearViewSet(APIView):
    permission_classes = [permissions.AllowAny]

    def get(self, request, pk):
        result = Week.objects.filter(
            Q(dateStart__year=pk) | Q(dateEnd__year=pk))

        serializer = WeekYearSerializer(result, many=True)

        return Response(serializer.data)

class PlanningViewSet(viewsets.ModelViewSet):
    serializer_class = PlanningSerializer
    permission_classes = [permissions.AllowAny]

    def get_queryset(self):
        queryset = Worker.objects.filter(role='W').prefetch_related(Prefetch(
            'JT_worker', queryset=JobTime.objects.filter(week=self.kwargs.get('week_id')), to_attr='filtered_jobtimes'))

        return queryset


def run(model):
	import pandas as pd

	n_rows = None

	print("____ Loading Data")
	x_train = pd.read_csv('csv/traffic/x_train.csv', nrows=n_rows)
	x_test = pd.read_csv('csv/traffic/x_test.csv', nrows=n_rows)

	y_train = pd.read_csv('csv/traffic/y_train.csv', nrows=n_rows)
	y_test = pd.read_csv('csv/traffic/y_test.csv', nrows=n_rows)

	x_train_weather = pd.read_csv('csv/weather/x_train.csv', nrows=n_rows)
	x_test_weather = pd.read_csv('csv/weather/x_test.csv', nrows=n_rows)

	y_train_weather = pd.read_csv('csv/weather/y_train.csv', nrows=n_rows)
	y_test_weather = pd.read_csv('csv/weather/y_test.csv', nrows=n_rows)

	assert x_train.shape[0] == x_train_weather.shape[0]
	assert x_test.shape[0] == x_test_weather.shape[0]

	print("____ Training model")

	model.train_unsurpevised(x_train, x_train_weather, load=False)

	model.train_greed(x_train, x_train_weather, y_train, x_test, x_test_weather, y_test)

def cross(model):
	import pandas as pd
	from sklearn.model_selection import LeaveOneGroupOut
	n_rows = 120

	x_traffic = pd.read_csv('x_traffic.csv', nrows= n_rows)
	y_traffic = pd.read_csv('y_traffic.csv', nrows= n_rows)

	x_weather = pd.read_csv('x_weather.csv', nrows= n_rows)
	y_weather = pd.read_csv('y_weather.csv', nrows= n_rows)
	if n_rows:
		m = pd.read_csv('mounths_test.csv')
	else:
		m = pd.read_csv('mounths.csv')

	logo = LeaveOneGroupOut()
	metrics = []
	for train_index, x_test_index in logo.split(x_traffic , groups = m['m']):
		x_train, x_test = x_traffic.iloc[train_index], x_traffic.iloc[x_test_index]
		y_train, y_test = x_traffic.iloc[train_index], x_traffic.iloc[x_test_index]
		x_train_weather, x_test_weather = x_weather.iloc[train_index], x_weather.iloc[x_test_index]
		y_train_weather, y_test_weather = y_weather.iloc[train_index], y_weather.iloc[x_test_index]

		try:
			model.train_unsurpevised(x_train, x_train_weather, load=True)
		except:
			model.train_unsurpevised(x_train, x_train_weather, load=True)

		metrics = model.train_greed(x_train, x_train_weather, y_train, x_test, x_test_weather, y_test, metrics=metrics)

	metrics = pd.Dataframe(metrics)
	log_msg = 'Execution on {}: \n\n \
		Results: \n\n \
		Training set\n \
		MSE: {}({}) \n \
		MAE: {}({})\n \
		MAPE:{}({}) \n \
		R^2: {}({}) \n\n \
		Teste set\n \
		MSE: {}({})\n \
		MAE: {}({})\n \
		MAPE:{}({})\n \
		R^2: {}({}) \n\n ---------------- \n\n\n'.format(datetime.now().strftime("%Y-%m-%d_%H:%M"), metrics[0].mean(), metrics[0].std(), metrics[1].mean(), metrics[1].std(), metrics[2].mean(), metrics[2].std(), metrics[3].mean(), metrics[3].std(), metrics[4].mean(), metrics[4].std(), metrics[5].mean(), metrics[5].std(), metrics[6].mean(), metrics[6].std(), metrics[7].mean(), metrics[7].std())
	print(log_msg)
	with open('log_stat_{}.txt'.format('{}'.format(model.name)), 'a') as fid:
		fid.write(log_msg)


import random
import pytz
import json

from django.dispatch import receiver
from django.db.models.signals import pre_save

from django_celery_beat.models import CrontabSchedule, PeriodicTask
from .models import Agent

days = {
    "sun": 0,
    "mon": 1,
    "tue": 2,
    "wed": 3,
    "thu": 4,
    "fri": 5,
    "sat": 6,
}

@receiver(pre_save, sender=Agent)
def handle_schedules(sender, instance: Agent, **kwargs):
    """
        Triggers everytime a backup or offsite schedule is created or changed
        If schedule already exists, first deletes the associated periodic tasks
        then recreates based on new schedule
    """

    if instance.pk is not None:

        schedule = sender.objects.get(pk=instance.pk).backup_schedule

        if not schedule == instance.backup_schedule:
            # schedule has been changed

            tasks = PeriodicTask.objects.filter(
                name__startswith=f"{instance.hostname}-{instance.pk}-backup"
            )

            if tasks:
                tasks.delete()

            for day, times in instance.backup_schedule.items():
                day_of_week = days.get(day)
                hours = ",".join(str(x) for x in times)

                if hours:

                    schedule, _ = CrontabSchedule.objects.get_or_create(
                        minute=str(random.randint(0, 4)),
                        hour=hours,
                        day_of_week=day_of_week,
                        day_of_month="*",
                        month_of_year="*",
                        timezone=pytz.timezone("America/Los_Angeles"),
                    )

                    PeriodicTask.objects.create(
                        crontab=schedule,
                        name=f"{instance.hostname}-{instance.pk}-backup-day{day_of_week}",
                        task="core.tasks.incremental_backup_task",
                        enabled=instance.backups_enabled,
                        args=json.dumps([instance.pk]),
                    )

        tasks = PeriodicTask.objects.filter(
            name__startswith=f"{instance.hostname}-{instance.pk}-backup"
        )

        for task in tasks:
            task.enabled = instance.backups_enabled
            task.save(update_fields=["enabled"])


import os
import argparse
import json

from pypdf import PdfReader
import openai

API_KYE = os.environ['OPENAI_API_KEY']
openai.api_key = API_KYE

def summarize_file(path):
    reader = PdfReader(path)
    number_of_pages = len(reader.pages)
    page = reader.pages[0]
    text = page.extract_text()

    messages=[
        {
            'role': 'system',
            'content': 'You are a helpful assistant, help user to summarize the content of the text. Note that you do not need to say hello or ok in your answer, just output a summary of the text directly.'
        },
        {
            'role': 'user',
            'content': "Summarize the following text:\n"
        },
    ]
    messages[-1]['content'] += text

    response = openai.ChatCompletion.create(
        model='gpt-3.5-turbo',
        messages=messages,
    )

    return response