from selenium.webdriver.common.by import By

class BasePageLocators(object):
    ETSY_LOGO = (By.XPATH, "//span[contains(@class, 'etsy-icon') and @id='logo']")

class SearchPageLocators(object):
    SEARCH_TEXTBOX = (By.XPATH, "//input[@data-id='search-query' and @name='search_query']")
    SEARCH_BUTTON = (By.XPATH, "//button[@type='submit' and @value='Search']")
    SEARCH_RESULTS = (By.XPATH, "//h1[contains(@class, 'wt-text-caption')]")
    TOP_NAV_CATEGORY_LINK = (By.XPATH, "//li[contains(@class, 'top-nav-item')]//a//span[contains(text(), 'Home & Living')]")


#!/bin/env python

import os
import logging
import shrub

logging.basicConfig(level=logging.INFO)

################# XXX update these env accordingly XXX #############

target_name = 'cpu'
network = 'mobilenet'
useLayout = 'NCHW'
# useLayout = 'NHWC'
dtype = 'float32'

useCores = 4
bigLITTLE = 1

####################################################################

target = shrub.tvm.TargetProvider(target_name, bigLITTLE, useCores)
net, params, model = shrub.tvm.get_model(network, dtype, useLayout)

tuner = shrub.tvm.Tuner(model.name, target)
tuner.tune(net, params, target)
dep = shrub.tvm.Deployable()
dep.build(target, model, net, params, tuner.record)
dep.export(target)

model.genInput()
model.store('input')
rti = shrub.tvm.RuntimeWrapper(model, target, dep)
rti.run(model, useLayout, dumpOutput=True)
# shrub.tflite.run(model, dumpOutput=True, logLevel=logging.INFO)
# rti.profile(target, 10)


"""Handle parsing and providing pipette information."""
import re
from typing import Dict, Tuple
import struct
from opentrons_shared_data.errors.exceptions import (
    InvalidInstrumentData,
    PythonException,
)
from opentrons_hardware.firmware_bindings.constants import PipetteName
from opentrons_hardware.instruments.serial_utils import ensure_serial_length

# Separate string into 3 named groups:
#   - name P any
#   - model
#   - code

RAW_SERIAL_STRING = (
    "^"  # start of string
    r"(?P<name>P[\w\d]{3})"  # "name" group starts with P and contains exactly 3 alphanumeric characters
    "V"  # The character V
    r"(?P<model>\d{2})"  # "model" group contains exactly 2 digits
    r"(?P<code>[\w\d]{0,12})"  # "code" group contains 0 to 12 inclusive alphanumeric characters
    "$"  # end of string
)
SERIAL_RE = re.compile(RAW_SERIAL_STRING)

NAME_LOOKUP: Dict[str, PipetteName] = {
    "P1KS": PipetteName.p1000_single,
    "P1KM": PipetteName.p1000_multi,
    "P50S": PipetteName.p50_single,
    "P50M": PipetteName.p50_multi,
    "P1KH": PipetteName.p1000_96,
    "P50H": PipetteName.p50_96,
}

SERIAL_FORMAT_MSG = (
    f'Serial numbers must have the format PNNNVMMXXXXXX... where NNN is one of {", ".join(NAME_LOOKUP.keys())}, '
    "MM is a two-digit model number, and the rest is some serial code."
)

def info_from_serial_string(serialval: str) -> Tuple[PipetteName, int, bytes]:
    """Parse pipette information from a serial code.

    The return value is the serial parts: the name, the model, and the serial.

    Note: this is not the inverse of serial_val_from_parts. The input should be
    something a human scans, not something returned from a pipette. A bytestring
    from a pipette is probably not valid ascii and even if it is, you won't get
    the expected values from passing it to this function.
    """
    matches = SERIAL_RE.match(serialval.strip())
    if not matches:
        raise InvalidInstrumentData(
            message=f"The serial number {serialval.strip()} is not valid. {SERIAL_FORMAT_MSG}",
            detail={"serial": serialval},
        )
    try:
        name = NAME_LOOKUP[matches.group("name")]
    except KeyError:
        raise InvalidInstrumentData(
            message=f"The pipette name part of the serial number ({matches.group('name')}) is unknown. {SERIAL_FORMAT_MSG}",
            detail={"name": matches.group("name")},
        )
    model = int(matches.group("model"))

    return (
        name,
        model,
        ensure_serial_length(matches.group("code").encode("ascii")),
    )

def serial_val_from_parts(name: PipetteName, model: int, serialval: bytes) -> bytes:
    """Encode pipette information into a bytestring suitable for EEPROM storage.

    Note: this is not the inverse of info_from_serial_string. info_from_serial_string
    parses information from a human-entered string that is probably the pipette barcode.
    This function takes that information and specifically encodes it in a bytestring for
    storage on the pipette, which includes binary data. If you call

    serial_val_from_parts(*info_from_serial_str(somestr.encode('ascii'))).decode('ascii')

    you will not get what you put in.
    """
    try:
        return struct.pack(
            ">HH16s",
            name.value,
            model,
            ensure_serial_length(serialval),
        )
    except struct.error as e:
        raise InvalidInstrumentData(
            message="Invalid pipette serial",
            detail={"name": name, "model": model, "serial": str(serialval)},
            wrapping=[PythonException(e)],
        )


"""Filters collection for gleitzsch."""
import sys
import random
import numpy as np
from skimage import exposure
from skimage import transform as tf
from skimage import filters
from skimage import color
from skimage.draw import polygon
# from modules.blur_detection import detect_blur

def eprint(line, end="\n"):
    """Like print but for stdout."""
    sys.stderr.write(line + end)

def parts(lst, n=25):
    """Split an iterable into list of iterables of size n."""
    return [lst[i:i + n] for i in iter(range(0, len(lst), n))]

def adjust_contrast(im, l_p, r_p):
    """Contrast correction."""
    perc_left, perc_right = np.percentile(im, (l_p, r_p))
    im = exposure.rescale_intensity(im, in_range=(perc_left, perc_right))
    return im

def xyz_shift(img, kt):
    """Apply chromatic aberration."""
    img = color.rgb2xyz(img)
    shp = img.shape
    red = img[:, :, 0]
    green = img[:, :, 1]
    blue = img[:, :, 2]
    # split channels, make shift
    red = tf.resize(red, output_shape=(shp[0], shp[1]))
    green = tf.resize(green, output_shape=(shp[0] - kt, shp[1] - kt))
    blue = tf.resize(blue, output_shape=(shp[0] - 2 * kt, shp[1] - 2 * kt))

    w, h = blue.shape
    ktd2 = int(kt / 2)
    red_n = np.reshape(red[kt: -kt, kt: -kt], (w, h, 1))
    green_n = np.reshape(green[ktd2: -1 * ktd2, ktd2: -1 * ktd2], (w, h, 1))
    blue_n = np.reshape(blue[:, :], (w, h, 1))

    new_im = np.concatenate((red_n, green_n, blue_n), axis=2)
    new_im = tf.resize(new_im, (shp[0], shp[1]))
    new_im = color.xyz2rgb(new_im)
    return new_im

def rgb_shift(img, kt):
    """Apply chromatic aberration."""
    shp = img.shape
    red = img[:, :, 0]
    green = img[:, :, 1]
    blue = img[:, :, 2]
    # split channels, make shift
    red = tf.resize(red, output_shape=(shp[0], shp[1]))
    green = tf.resize(green, output_shape=(shp[0] - kt, shp[1] - kt))
    blue = tf.resize(blue, output_shape=(shp[0] - 2 * kt, shp[1] - 2 * kt))

    w, h = blue.shape
    ktd2 = int(kt / 2)
    red_n = np.reshape(red[kt: -kt, kt: -kt], (w, h, 1))
    green_n = np.reshape(green[ktd2: -1 * ktd2, ktd2: -1 * ktd2], (w, h, 1))
    blue_n = np.reshape(blue[:, :], (w, h, 1))

    new_im = np.concatenate((red_n, green_n, blue_n), axis=2)
    new_im = tf.resize(new_im, (shp[0], shp[1]))
    return new_im

def __bayer(im):
    """Make monochrome pixels."""
    h, w, d = im.shape
    for i in range(h):
        for j in range(w):
            channels_to_rm = np.random.choice(range(3), size=2, replace=False)
            im[i][j][channels_to_rm[0]] = 0.0
            im[i][j][channels_to_rm[1]] = 0.0
    return im

def bayer(im):
    """It is not a bayer filter anymore actually."""
    w, h, d = im.shape
    imlen = w * h * d
    pix_num = w * h
    chan_numbers = np.random.choice([0, 1, 2], pix_num)
    im_flat, new_im_flat = np.reshape(im, imlen), []
    # new_im_flat = [im_flat[i] if yes_not[i] == 1 else 0.0 for i in range(imlen)]
    for num, pixel in enumerate(parts(im_flat, n=3)):
        color = [0.0, 0.0, 0.0]
        chan_num = chan_numbers[num]
        color[chan_num] = pixel[chan_num]
        new_im_flat.extend(color)
    print(len(new_im_flat))
    new_im = np.reshape(np.array(new_im_flat), (w, h, d))
    return new_im

def _bayer(im):
    """It is not a bayer filter anymore actually."""
    w, h, d = im.shape
    imlen = w * h * d
    # yes_not = np.random.choice([0, 1], imlen, p=[0.1, 0.9])
    chan_numbers = np.random.choice([0, 1, 2], imlen)
    im_flat, new_im_flat = np.reshape(im, imlen), []
    # new_im_flat = [im_flat[i] if yes_not[i] == 1 else 0.0 for i in range(imlen)]
    for i in range(imlen):
        chan = chan_numbers[i]
        if chan == 0:
            color = [im_flat[0], 0, 0]
        elif chan == 1:
            color = [0, im_flat[1], 0]
        else:
            color = [0, 0, im_flat[2]]
        new_im_flat.extend(color)
    new_im = np.reshape(np.array(new_im_flat), (w, h, d))
    return new_im

def amplify(im):
    """Self-overlap."""
    REPEATS = 0
    shift = int(np.random.uniform(low=30, high=130))
    sign = np.random.choice([-1, 1], 1)[0]
    shift *= sign
    # print(shift)
    delim, kt = 1, 3
    layer_sh = np.roll(a=im, axis=1, shift=shift) / kt
    im += layer_sh
    delim += 1 / kt
    kt /= 3

    for _ in range(REPEATS):
        layer_sh = np.roll(a=layer_sh, axis=1, shift=shift) / kt
        im += layer_sh
        delim += 1 / kt
        kt /= 3

    im /= delim
    # im[im > 1] = 1.0
    return im

def glitter(im, alen=300):
    """Make glitter."""
    dots = []
    w, h, _ = im.shape
    for _ in range(alen):
        dx = random.choice(range(w))
        dy = random.choice(range(h))
        dots.append((dx, dy))
    for dot in dots:
        try:
            im[dot[0] - 1: dot[0], dot[1] - 3: dot[1] + 3, :] = 1
        except IndexError:
            pass
    return im

def color_shift(im, val):
    """Change color throw HSV space."""
    im_hsv = color.rgb2hsv(im)
    hue = im_hsv[:, :, 0]
    hue += val
    hue[hue > 1] = hue[hue > 1] - 1
    hue[hue < 0] = hue[hue < 0] + 1
    im_hsv[:, :, 0] = hue
    im_rgb = color.hsv2rgb(im_hsv)
    return im_rgb

def black(im, thr):
    """Return two-colored bw image."""
    # get pixel sum (one number) instead of pixel (3 nums)
    col_sum = np.sum(im, axis=2)
    black_im = np.zeros((col_sum.shape[0], col_sum.shape[1]))
    # 0 - less than thr, 1 - more than
    black_im[col_sum > thr] = 1
    black_im = np.reshape(black_im, (black_im.shape[0], black_im.shape[1], 1))
    return np.concatenate((black_im, black_im, black_im), axis=2)

def remove_whites(im):
    """Diminish colors that are brighter than a threshold set. For an image."""
    col_sum = np.sum(im, axis=2)
    over_thr = np.reshape((col_sum - 2), (im.shape[0], im.shape[1], 1))
    over_thr = np.concatenate((over_thr, over_thr, over_thr), axis=2)
    new_im = np.where(over_thr > 0, im - over_thr, im)
    new_im[new_im < 0.0] = 0.0
    new_im[new_im > 1.0] = 1.0
    return new_im

def rainbow_layer(im):
    new_im = rgb_shift(im, kt=88)
    new_im = filters.gaussian(new_im, sigma=30, multichannel=True, mode='reflect', cval=0.6)
    img_hsv = color.rgb2hsv(new_im)
    img_hsv[..., 1] *= 3
    img_hsv[..., 2] *= 1.4
    img_hsv[img_hsv >= 1.0] = 1.0
    new_im = color.hsv2rgb(img_hsv)
    new_im = remove_whites(new_im)
    return new_im

def make_rainbow(im):
    """Apply rainbow filter."""
    black_instance = black(im, thr=2.1)
    rainbow_pic = rainbow_layer(black_instance)
    rainbow_pic /= 2
    new_arr = rainbow_pic + im
    new_arr[new_arr > 1.0] = 1.0
    return new_arr

def interlace(im, zero_prob=0.9):
    """Add interlacing fields."""
    w, h, d = im.shape
    coeff, processed = 3, []
    shift_wide = 2

    half_prob = (1 - zero_prob) / 2
    half_probs = [half_prob / shift_wide for _ in range(shift_wide)]
    shift_probs = half_probs + [zero_prob] + half_probs
    shif_pos = list(range(- shift_wide, shift_wide + 1))
    shift = 0

    for num, i in enumerate(range(0, w, coeff)):
        row = im[i: i + coeff + 1, :, :]
        row = row / 1.35 if num % 2 == 0 else row
        shift_p = np.random.choice(shif_pos, 1, p=shift_probs)[0]
        shift += shift_p
        row = np.roll(a=row, axis=0, shift=shift)  # small part
        row = np.roll(a=row, axis=1, shift=shift)  # long size
        processed.append(row)

    merge = np.concatenate(processed, axis=0)
    merge = tf.resize(merge, (w, h))
    return merge

def add_vertical(image):
    """Hard to say."""
    first_row = np.reshape(image[0, :, :], newshape=(1, image.shape[1], image.shape[2]))
    stretch = np.repeat(first_row, image.shape[0], axis=0)
    add = image + stretch / 10
    add[add > 1] = 1.0
    return add

def vert_streaks(im):
    """Add some vertical streaks on image."""
    w, h, d = im.shape
    processed = []
    streaks_borders_num = random.choice(range(6, 16, 2))
    streaks_borders = [0] + list(sorted(np.random.choice(range(h), streaks_borders_num, replace=False))) + [h]
    for num, border in enumerate(streaks_borders[1:]):
        prev_border = streaks_borders[num]
        pic_piece = im[:, prev_border: border, :]
        if num % 2 != 0:  # don't touch this part
            processed.append(pic_piece)
            continue
        piece_h, piece_w, _ = pic_piece.shape
        piece_rearranged = []
        shifts_raw = sorted([i if i > 0 else -i for i in map(int, np.random.normal(5, 10, piece_w))])
        shifts_add = np.random.choice(range(-5, 2), piece_w)
        shifts_mod = [shifts_raw[i] + shifts_add[i] for i in range(piece_w)]
        shifts_left = [shifts_mod[i] for i in range(0, piece_w, 2)]
        shifts_right = sorted([shifts_mod[i] for i in range(1, piece_w, 2)], reverse=True)
        shifts = shifts_left + shifts_right
        for col_num, col_ind in enumerate(range(piece_w)):
            col = pic_piece[:, col_ind: col_ind + 1, :]
            col = np.roll(col, axis=0, shift=shifts[col_num])
            piece_rearranged.append(col)
        piece_shifted = np.concatenate(piece_rearranged, axis=1)
        processed.append(piece_shifted)
    new_im = np.concatenate(processed, axis=1)
    new_im = tf.resize(new_im, (w, h))
    return new_im

def horizonal_shifts(im, colorized=True):
    """Add random horizontal shifts."""
    w, h, d = im.shape
    processed = []
    shifts_borders_num = random.choice(range(6, 16, 2))
    shifts_borders = [0] + list(sorted(np.random.choice(range(w), shifts_borders_num, replace=False))) + [h]
    for num, border in enumerate(shifts_borders[1:]):
        prev_border = shifts_borders[num]
        pic_piece = im[prev_border: border, :, :]
        shift = 0 if num % 2 != 0 else random.choice(range(40))
        shifted = np.roll(pic_piece, shift=shift, axis=1)
        shifted = shifted if not colorized else np.roll(shifted, shift=shift, axis=2)
        processed.append(shifted)
    new_im = np.concatenate(processed, axis=0)
    new_im = tf.resize(new_im, (w, h))
    return new_im

def _vert_streaks(im, kernel=1, partial=False):
    """Add some vertical streaks on image."""
    w, h, d = im.shape
    processed = []
    shifts = np.random.choice([-2, -1, 0, 1, 2], h)
    current_shift = 0
    change_points = np.random.choice([False, True], h, p=[0.97, 0.03]) if partial else [0 for _ in range(h)]
    point = True
    for num, i in enumerate(range(0, h, kernel)):
        change_ = change_points[num]
        point = not point if change_ else point
        col = im[:, i: i + kernel + 1, :]
        if not point:
            processed.append(col)
            current_shift = 0
            continue
        current_shift += shifts[num]
        col = np.roll(col, axis=0, shift=current_shift)
        processed.append(col)
    merge = np.concatenate(processed, axis=1)
    merge = tf.resize(merge, (w, h))
    return merge

def add_magic(im):
    """Add some magic."""
    eprint("detecting blur...")
    w, h, d = im.shape
    blur_map_layer = np.reshape(detect_blur(im, kernel_size=8), (w, h, 1))
    blur_map_layer = filters.gaussian(blur_map_layer, sigma=5)
    blur_map = np.concatenate((blur_map_layer, blur_map_layer, blur_map_layer), axis=2)
    blur_map[blur_map < 0] = 0.0
    blur_map[blur_map > 1] = 1.0
    eprint("blur detected")
    magic_layer = im
    magic_layer = rgb_shift(magic_layer, kt=28)
    magic_layer = vert_streaks(magic_layer)
    magic_layer = filters.gaussian(magic_layer, sigma=3)
    magic_layer -= blur_map
    magic_layer[magic_layer < 0] = 0.0
    im += magic_layer
    im[im > 1] = 1.0
    return im

def add_figs(im):
    """Add random shapes."""
    figs = np.zeros((im.shape[0], im.shape[1], 3))
    wide = np.random.choice(range(10, 200), 1)[0]
    poly = np.array((
        (0, 0),
        (0, wide),
        (1000, wide),
        (1000, 0),
    ))
    rr, cc = polygon(poly[:, 0], poly[:, 1], figs.shape)
    figs[rr, cc, 0] = 0.7
    figs[rr, cc, 1] = 0.1
    figs[rr, cc, 2] = 0.2
    figs = np.roll(figs, shift=np.random.choice(range(1000), 1)[0], axis=1)
    figs = filters.gaussian(figs, sigma=5, multichannel=True, mode='reflect', cval=0.6)
    im += figs
    im[im > 1.0] = 1.0
    return im

def reconstruct(im, kHz):
    """Reconstruct shifted rows."""
    if kHz == 16.0:
        return im
    w, h, d = im.shape
    processed = []
    for num, i in enumerate(range(0, w, 1)):
        row = im[i: i + 1, :, :]
        if kHz == 16.01:
            shift = int(num * 0.5)
        elif kHz == 15.99:
            shift = int(num * -0.5)
        elif kHz == 15.98:
            shift = int(num * -1)
        else:
            shift = 0
        row = np.roll(a=row, axis=1, shift=shift)
        processed.append(row)
    merge = np.concatenate(processed, axis=0)
    merge = tf.resize(merge, (w, h))
    return merge

def increase_saturation(im):
    """Increase saturation."""
    hsl_im = color.rgb2hsv(im)
    hsl_im[:, :, 1] *= 1.45
    hsl_im[hsl_im > 1] = 1.0
    im = color.hsv2rgb(hsl_im)
    return im


import os
import time
import wandb
import torch
import envpool
import numpy as np
import torch.optim as optim
from util.env_wrappers import VecAdapter
from stable_baselines3.common.vec_env import VecMonitor
from stable_baselines3.common.evaluation import evaluate_policy

def lerp(start, end, weight):
    return start + weight * (end - start)

def num_cpus():
    try:
        return len(os.sched_getaffinity(0))
    except AttributeError:
        return os.cpu_count()

def get_eval_env(env_id, env_type, num_envs, seed, num_threads=None):
    if num_threads is None:
        num_threads = num_cpus()
    else:
        num_threads = min(num_threads, num_cpus())
    eval_env = envpool.make(env_id, env_type=env_type, num_envs=num_envs, seed=seed, num_threads=num_threads)
    eval_env.spec.id = env_id
    eval_env = VecAdapter(eval_env)
    eval_env = VecMonitor(eval_env)
    return eval_env

def linear_schedule(start_e: float, end_e: float, duration: int, t: int):
    slope = (end_e - start_e) / duration
    return max(slope * t + start_e, end_e)

def num_train_steps(global_steps, num_envs, samples_step, batch_size):
    if num_envs > samples_step:
        return True, int(num_envs*samples_step/batch_size)
    else:
        return global_steps % int(batch_size/samples_step) == 0, 1

@torch.no_grad()
def evaluate_sb3(model, env, eval_eps, track, step=None, prefix='', log_time=True):
    start_time = time.time()
    rewards, lengths = evaluate_policy(model, env, n_eval_episodes=eval_eps, return_episode_rewards=True)
    finish_time = time.time()
    mean_reward, std_reward = np.mean(rewards), np.std(rewards)
    mean_ep_length, std_ep_length = np.mean(lengths), np.std(lengths)
    fps = int(np.sum(lengths) / (finish_time - start_time))
    if track:
        log = {f"eval/{prefix}mean_reward": mean_reward,
               f"eval/{prefix}stdev_reward": std_reward,
               f"eval/{prefix}mean_ep_length": mean_ep_length,
               f"eval/{prefix}stdev_ep_length": std_ep_length}
        if log_time: log.update({f"time/{prefix}eval_fps": fps})
        wandb.log(log, step=step)

    return mean_reward, std_reward, mean_ep_length, std_ep_length, fps

def get_optimizer(opt_name, parameters, lr, wd, eps):
    if opt_name == 'Adam':
        optimizer = optim.Adam(parameters, lr=lr, weight_decay=wd, eps=eps)
    elif opt_name == 'AdamW':
        optimizer = optim.AdamW(parameters, lr=lr, weight_decay=wd, eps=eps)
    elif opt_name == 'SGD':
        optimizer = optim.SGD(parameters, lr=lr, weight_decay=wd)
    elif opt_name == 'RMSprop':
        optimizer = optim.RMSprop(parameters, lr=lr, weight_decay=wd, eps=eps)
    else:
        raise ValueError(f'Invalid optimizer, {opt_name} must be one of Adam, AdamW, SGD, or RMSprop')
    return optimizer


import evdev
from evdev import InputDevice, categorize, ecodes

class File(object):
  def __init__(self, path):
    self.path = path

  def listen(self):
    with open(self.path, 'r') as fp:
      for line in fp.readlines():
          yield line.strip()

# Provided as an example taken from my own keyboard attached to a Centos 6 box:
scancodes = {
    # Scancode: ASCIICode
    0: None, 1: u'ESC', 2: u'1', 3: u'2', 4: u'3', 5: u'4', 6: u'5', 7: u'6', 8: u'7', 9: u'8',
    10: u'9', 11: u'0', 12: u'-', 13: u'=', 14: u'BKSP', 15: u'TAB', 16: u'q', 17: u'w', 18: u'e', 19: u'r',
    20: u't', 21: u'y', 22: u'u', 23: u'i', 24: u'o', 25: u'p', 26: u'[', 27: u']', 28: u'CRLF', 29: u'LCTRL',
    30: u'a', 31: u's', 32: u'd', 33: u'f', 34: u'g', 35: u'h', 36: u'j', 37: u'k', 38: u'l', 39: u';',
    40: u'"', 41: u'`', 42: u'LSHFT', 43: u'\\', 44: u'z', 45: u'x', 46: u'c', 47: u'v', 48: u'b', 49: u'n',
    50: u'm', 51: u',', 52: u'.', 53: u'/', 54: u'RSHFT', 56: u'LALT', 57: u' ', 100: u'RALT'
}

capscodes = {
    0: None, 1: u'ESC', 2: u'!', 3: u'@', 4: u'#', 5: u'$', 6: u'%', 7: u'^', 8: u'&', 9: u'*',
    10: u'(', 11: u')', 12: u'_', 13: u'+', 14: u'BKSP', 15: u'TAB', 16: u'Q', 17: u'W', 18: u'E', 19: u'R',
    20: u'T', 21: u'Y', 22: u'U', 23: u'I', 24: u'O', 25: u'P', 26: u'{', 27: u'}', 28: u'CRLF', 29: u'LCTRL',
    30: u'A', 31: u'S', 32: u'D', 33: u'F', 34: u'G', 35: u'H', 36: u'J', 37: u'K', 38: u'L', 39: u':',
    40: u'\'', 41: u'~', 42: u'LSHFT', 43: u'|', 44: u'Z', 45: u'X', 46: u'C', 47: u'V', 48: u'B', 49: u'N',
    50: u'M', 51: u'<', 52: u'>', 53: u'?', 54: u'RSHFT', 56: u'LALT',  57: u' ', 100: u'RALT'
}
#setup vars
x = ''
caps = False

#grab provides exclusive access to the device

class Device(object):
  def __init__(self, device):
    self.dev = InputDevice(device)
    self.dev.grab()

  def listen(self):
    #loop
    x = ''
    caps = False
    for event in self.dev.read_loop():
      if event.type == ecodes.EV_KEY:
        data = categorize(event)  # Save the event temporarily to introspect it
        if data.scancode == 42:
          if data.keystate == 1:
            caps = True
          if data.keystate == 0:
            caps = False

        if data.keystate == 1:  # Down events only
          if caps:
            key_lookup = u'{}'.format(capscodes.get(data.scancode)) or u'UNKNOWN:[{}]'.format(data.scancode)  # Lookup or return UNKNOWN:XX
          else:
            key_lookup = u'{}'.format(scancodes.get(data.scancode)) or u'UNKNOWN:[{}]'.format(data.scancode)  # Lookup or return UNKNOWN:XX
          if (data.scancode != 42) and (data.scancode != 28):
            x += key_lookup

          if(data.scancode == 28):
            yield x
            x = ''


import numpy as np
import tensorflow as tf

import matplotlib.pyplot as plt
import matplotlib.patches as patches

def image_to_np_array(image):
    (im_width, im_height) = image.size
    return np.array(image.getdata()).reshape((im_height, im_width, 3)).astype(np.uint8)

def load_model_to_session(sess, pb_model_path):
    od_graph_def = sess.graph_def
    with tf.gfile.GFile(pb_model_path, 'rb') as fid:
        serialized_graph = fid.read()
        od_graph_def.ParseFromString(serialized_graph)
        tf.import_graph_def(od_graph_def, name='')

def load_pbtxt_to_label_dict(pb_label_file_path):  # only work for mascoco label map now
    label_dict = dict()
    with open(pb_label_file_path) as f:
        content = f.readlines()
        content = [x.strip() for x in content]
        for i in range(len(content) // 5):
            index = int(content[i * 5 + 2][4:])
            name = content[i * 5 + 3][15:-1]
            label_dict[index] = name
    return label_dict

def inference_one_image(sess, image):
    graph = sess.graph
    image_tensor = graph.get_tensor_by_name('image_tensor:0')  # input
    feed_dict = {image_tensor: image}
    ops = graph.get_operations()
    all_tensor_names = {output.name for op in ops for output in op.outputs}
    tensor_dict = dict()
    for key in ['num_detections', 'detection_boxes', 'detection_scores', 'detection_classes', 'detection_masks']:
        tensor_name = key + ':0'
        if tensor_name in all_tensor_names:
            tensor_dict[key] = graph.get_tensor_by_name(tensor_name)
    output_dict = sess.run(tensor_dict, feed_dict=feed_dict)
    output_dict['num_detections'] = int(output_dict['num_detections'][0])
    output_dict['detection_classes'] = output_dict['detection_classes'][0].astype(np.int64)
    output_dict['detection_boxes'] = output_dict['detection_boxes'][0]
    output_dict['detection_scores'] = output_dict['detection_scores'][0]
    return output_dict

def draw_bounding_boxes_on_image_array(image_np, boxes, classes, scores, min_score_threshold=0.5, label_dict=None):
    height = image_np.shape[0]
    width = image_np.shape[1]

    # Create figure and axes
    fig, ax = plt.subplots()
    ax.imshow(image_np)
    for i in range(boxes.shape[0]):
        if scores[i] < min_score_threshold:
            continue
        box = tuple(boxes[i].tolist())
        y0 = int(box[0]*height)
        x0 = int(box[1]*width)
        y1 = int(box[2]*height)
        x1 = int(box[3]*width)
        # Create a Rectangle patch
        rect = patches.Rectangle((x0, y0), x1 - x0, y1 - y0, linewidth=1, edgecolor='r', facecolor='none', fill=False)
        ax.add_patch(rect)
        if label_dict is not None:
            display_str = label_dict[classes[i]] + ', ' + str(round(scores[i]*100))
            ax.text(x0 ,y0, display_str, color='white')
    plt.show()


"""Provide functions to modify model components."""

from ast import NodeTransformer
from itertools import chain
from typing import TYPE_CHECKING, Dict

from ..core.gene import ast2str
from .delete import get_compiled_gene_reaction_rules

if TYPE_CHECKING:
    from cobra import Gene, Model

# Set of tuples of operators and their corresponding textual form
_renames = (
    (".", "_DOT_"),
    ("(", "_LPAREN_"),
    (")", "_RPAREN_"),
    ("-", "__"),
    ("[", "_LSQBKT"),
    ("]", "_RSQBKT"),
    (",", "_COMMA_"),
    (":", "_COLON_"),
    (">", "_GT_"),
    ("<", "_LT"),
    ("/", "_FLASH"),
    ("\\", "_BSLASH"),
    ("+", "_PLUS_"),
    ("=", "_EQ_"),
    (" ", "_SPACE_"),
    ("'", "_SQUOT_"),
    ('"', "_DQUOT_"),
)

def _escape_str_id(id_str: str) -> str:
    """Make a single string ID SBML compliant.

    Parameters
    ----------
    id_str: str
        The ID string to operate on.

    Returns
    -------
    str
        The SBML compliant ID string.

    """
    for c in ("'", '"'):
        if id_str.startswith(c) and id_str.endswith(c) and id_str.count(c) == 2:
            id_str = id_str.strip(c)
    for char, escaped_char in _renames:
        id_str = id_str.replace(char, escaped_char)
    return id_str

class _GeneEscaper(NodeTransformer):
    """Class to represent a gene ID escaper."""

    def visit_Name(self, node: "Gene") -> "Gene":
        """Escape string ID.

        Parameters
        ----------
        node: cobra.Gene
            The gene object to work on.

        Returns
        -------
        cobra.Gene
            The gene object whose ID has been escaped.

        """
        node.id = _escape_str_id(node.id)
        return node

def escape_ID(model: "Model") -> None:
    """Make all model component object IDs SBML compliant.

    Parameters
    ----------
    model: cobra.Model
        The model to operate on.

    """
    for x in chain([model], model.metabolites, model.reactions, model.genes):
        x.id = _escape_str_id(x.id)
    model.repair()
    gene_renamer = _GeneEscaper()
    for rxn, rule in get_compiled_gene_reaction_rules(model).items():
        if rule is not None:
            rxn._gene_reaction_rule = ast2str(gene_renamer.visit(rule))

class _Renamer(NodeTransformer):
    """
    Class to represent a gene renamer.

    Parameters
    ----------
    rename_dict: dict of {str: str}
        The dictionary having keys as old gene names
        and value as new gene names.

    """

    def __init__(self, rename_dict: Dict[str, str], **kwargs) -> None:
        """Initialize a new object.

        Other Parameters
        ----------------
        **kwargs:
            Further keyword arguments are passed on to the parent class.

        """
        super().__init__(**kwargs)
        self.rename_dict = rename_dict

    def visit_Name(self, node: "Gene") -> "Gene":
        """Rename a gene.

        Parameters
        ----------
        node: cobra.Gene
            The gene to rename.

        Returns
        -------
        cobra.Gene
            The renamed gene object.

        """
        node.id = self.rename_dict.get(node.id, node.id)
        return node

def rename_genes(model: "Model", rename_dict: Dict[str, str]) -> None:
    """Rename genes in a model from the `rename_dict`.

    Parameters
    ----------
    model: cobra.Model
        The model to operate on.
    rename_dict: dict of {str: str}
        The dictionary having keys as old gene names
        and value as new gene names.

    """
    recompute_reactions = set()  # need to recompute related genes
    remove_genes = []
    for old_name, new_name in rename_dict.items():
        # undefined if there a value matches a different key
        try:
            gene_index = model.genes.index(old_name)
        except ValueError:
            gene_index = None
        old_gene_present = gene_index is not None
        new_gene_present = new_name in model.genes
        if old_gene_present and new_gene_present:
            old_gene = model.genes.get_by_id(old_name)
            # Added in case not renaming some genes:
            if old_gene is not model.genes.get_by_id(new_name):
                remove_genes.append(old_gene)
                recompute_reactions.update(old_gene._reaction)
        elif old_gene_present and not new_gene_present:
            # rename old gene to new gene
            gene = model.genes[gene_index]
            # trick DictList into updating index
            model.genes._dict.pop(gene.id)  # ugh
            gene.id = new_name
            model.genes[gene_index] = gene
        elif not old_gene_present and new_gene_present:
            pass
        else:  # if not old gene_present and not new_gene_present
            # the new gene's _model will be set by repair
            # This would add genes from rename_dict
            # that are not associated with a rxn
            # cobra_model.genes.append(Gene(new_name))
            pass
    model.repair()

    gene_renamer = _Renamer(rename_dict)
    for rxn, rule in get_compiled_gene_reaction_rules(model).items():
        if rule is not None:
            rxn._gene_reaction_rule = ast2str(gene_renamer.visit(rule))

    for rxn in recompute_reactions:
        rxn.gene_reaction_rule = rxn._gene_reaction_rule
    for i in remove_genes:
        model.genes.remove(i)


import json
import re
import pytz
import discord
import requests
import utils.global_variables as gv
from loguru import logger
from bs4 import BeautifulSoup
from datetime import datetime
from discord.ext.commands import Bot
from discord.ext import tasks, commands

class ThreadTasks(commands.Cog):
    def __init__(self, bot):
        self.index = 0
        self.bot: Bot = bot
        self.news_nasa.start()

    def cog_unload(self):
        self.news_nasa.cancel()

    @tasks.loop(minutes=1.0)
    @logger.catch
    async def news_nasa(self):
        guild_id = 484402073744703498
        channel_id = 687285704849489935

        if gv.TestBot:
            guild_id = 435485527156981770
            channel_id = 531541696995917844

        date = datetime.now().date()
        time = datetime.now(pytz.timezone('Europe/Moscow')).time()
        old_date = gv.DataBaseClass.get_value('date_post_nasa_news', None)

        logger.debug(f'nasa_news: guild_id - {guild_id}, channel_id - {channel_id}, '
                     f'status - {"on" if gv.ISPostNasaNews else "off" }, '
                     f'data - {date} '
                     f'time - {time} '
                     f'old_date - {old_date}')

        if not gv.ISPostNasaNews:
            logger.debug('NASA news - Off')
            return

        if not gv.NowPostNasaNews and old_date is not None and (date <= old_date or time.hour != 8):
            logger.debug('Start task news_nasa - Off')
            return

        if gv.NowPostNasaNews:
            gv.NowPostNasaNews = False

        logger.info('Start task news_nasa - On')
        gv.DataBaseClass.set_value('date_post_nasa_news', stype='date', value=date)

        guild: discord.Guild = self.bot.get_guild(guild_id)
        channel: discord.TextChannel = guild.get_channel(channel_id)

        response = requests.get('https://apod.nasa.gov/apod/')
        if response.status_code == 200:
            text = response.text

            soup = BeautifulSoup(text, "lxml")
            body = soup.body
            href = body.find_all('a', href=re.compile(r'image/\d{4}'))[0]
            min_src = href.img['src']

            description = str(body.find_all('p')[2])[26:-5]
            description = description.replace('\n', '')
            description = re.sub(r'(.+?)', r'\1', description)

            replaces = [
                (' .', '. '),
                (' ,', ', '),
                ('.', '. '),
                (',', ', '),
                ('  ', ' ')
            ]

            for repl_item in replaces:
                description = description.replace(*repl_item)

            ru_description = self.translate(description)

            emd = discord.Embed(color=gv.AdminColor)
            emd.add_field(name='**Description**', value=description)
            if ru_description:
                emd.add_field(name='**Описание**', value=ru_description, inline=True)
            emd.add_field(name='**Ссылка на большую версию**', value=f'https://apod.nasa.gov/apod/{href["href"]}',
                          inline=False)
            emd.set_image(url=f'https://apod.nasa.gov/apod/{min_src}')
            emd.set_footer(text=guild.name, icon_url=guild.icon.url)
            await channel.send(embed=emd)

    @news_nasa.before_loop
    async def before_news_nasa(self):
        logger.info('await connect bot')
        await self.bot.wait_until_ready()

    def translate(self, text: str):
        if not text:
            return
        body = {
            "targetLanguageCode": 'ru',
            "texts": text
        }

        headers = {
            "Content-Type": "application/json",
            "Authorization": "Api-Key {0}".format(gv.IAM_TOKEN)
        }

        response = requests.post('https://translate.api.cloud.yandex.net/translate/v2/translate',
                                 json=body,
                                 headers=headers
                                 )

        try:
            js = json.loads(response.text)
            if 'code' in js:
                logger.error(f'Request error: {response.text}')
                return ''
            return js.get('translations', [dict()])[0].get('text', '')
        except Exception as e:
            logger.error(f'Request error: {e}')
            return ''

async def setup(bot):
    await bot.add_cog(ThreadTasks(bot))


import logging
import os
import sys

from collections import namedtuple
import dill
import numpy as np
import tensorflow as tf

from dgm4nlp.annealing import AnnealingSchedule
from dgm4nlp.nlputils import AERSufficientStatistics
from dgm4nlp.nlputils import Multitext
from dgm4nlp.nlputils import class_distribution
from dgm4nlp.tf.bilingual import prepare_test
from dgm4nlp.tf.bilingual import prepare_training
from dgm4nlp.tf.bilingual import prepare_validation
from dgm4nlp.tf.opt import minimise
from dgm4nlp.tf.ssoftmax import np_get_support
from dgm4nlp.tf.checkpoint import ModelTracker

from embedalign.model import EmbedAlignModel
from embedalign.model import VariationalApproximationSpecs
from embedalign.viterbi import get_viterbi

class EmbedAlignTrainer:
    """
    Takes care of training a model with SGD.
    """

    def __init__(self, model: EmbedAlignModel,
                 training, val, val_wa, test, test_wa,
                 output_dir,
                 # architecture
                 nb_softmax_samples=[0, 0],
                 softmax_approximation=['botev-batch', 'botev-batch'],
                 softmax_q_inv_temperature=[0., 0.],
                 # optimisation
                 num_epochs=5,
                 batch_size=16,
                 optimizer=tf.train.RMSPropOptimizer(learning_rate=0.001),
                 grad_clipping=None,
                 # annealing step
                 annealing_s=AnnealingSchedule(),
                 annealing_z=AnnealingSchedule(),
                 # tensorflow
                 session=None):
        """

        :param model: EmbedAlignModel
        :param training: training Multitext
        :param val: validation Multitext
        :param val_wa: validation manual alignments
        :param test: test Multitext
        :param test_wa: test manual alignments
        :param output_dir: where to save files
        :param nb_softmax_samples: enable sampled softmax approximations for P(X|z) and P(Y|x_a,z_a)
        :param softmax_approximation: choose type of sampled softmax approximations for P(X|z) and P(Y|x_a,z_a)
        :param softmax_q_inv_temperature: choose temperature for proxy in 'botev-batch' approximation
        :param num_epochs: maximum number of epochs
        :param batch_size: maximum samples in a batch
        :param optimizer: a tf optimiser
        :param grad_clipping: configure gradient clipping with a pair [floor, ceil]
        :param annealing_s: configure annealing for KL term associated with prior P(S)
        :param annealing_z: configure annealing for KL term associated with prior P(Z|s)
        :param session: a tf session
        """

        self.model = model
        self.session = session
        self.tracker = ModelTracker(session, model, output_dir)

        # Configure softmax approximations
        self.nb_softmax_samples = nb_softmax_samples
        self.softmax_approximation = softmax_approximation
        self.softmax_q_inv_temperature = softmax_q_inv_temperature
        #  for X
        if softmax_q_inv_temperature[0] > 0:
            logging.info('Computing x-side class frequencies (inverse temperature %f)', softmax_q_inv_temperature[0])
            self.freq_x = class_distribution(self.training, stream=0, batch_size=1000, normalise=False)
            self.freq_x **= softmax_q_inv_temperature[0]
        else:
            self.freq_x = None
        #  for Y
        if softmax_q_inv_temperature[1] > 0:
            logging.info('Computing y-side class frequencies (inverse temperature %f)', softmax_q_inv_temperature[1])
            self.freq_y = class_distribution(self.training, stream=1, batch_size=1000, normalise=False)
            self.freq_y **= softmax_q_inv_temperature[1]
        else:
            self.freq_y = None

        # Configure experiment
        os.makedirs(output_dir, exist_ok=True)
        self.training = training
        self.val = val
        self.val_wa = val_wa
        self.output_dir = output_dir
        self.test = test
        self.test_wa = test_wa

        # Configure loss
        self.annealing_s = annealing_s
        self.annealing_z = annealing_z

        # Configure optimiser
        self.num_epochs = num_epochs
        self.batch_size = batch_size
        self.optimizer = optimizer
        self.grad_clipping = grad_clipping
        self._build_optimizer()

    def _build_optimizer(self):
        """Buid the optimizer."""
        self.optimizer_state = minimise(
            loss=self.model.loss,
            optimizer=self.optimizer,
            grad_clipping=self.grad_clipping
        )

    def get_generator(self, corpus: Multitext, training_phase=None):
        if training_phase:
            generator = corpus.batch_iterator(
                batch_size=self.batch_size,
                endless=False,
                shorter_batch='trim',
                dynamic_sequence_length=True
            )
        else:
            generator = corpus.batch_iterator(
                batch_size=1,
                endless=False,
                shorter_batch='trim',
                dynamic_sequence_length=True
            )
        for batch in generator:
            (x, mx), (y, my) = batch
            # input to the TF graph
            is_training = corpus is self.training
            feed_dict = {
                self.model.x: x,
                self.model.y: y,
                self.model.training_phase: is_training,
                self.model.alpha_s: self.annealing_s.alpha(),
                self.model.alpha_z: self.annealing_z.alpha()
            }

            # additional inputs for approximations

            # X side
            vx = self.training.vocab_size(0)
            if 0 < self.nb_softmax_samples[0] < vx and self.softmax_approximation[0] == 'botev-batch':  # in case we are employing a sampled softmax
                support_x, importance_x, nb_probable_x = np_get_support(  # we sample the support using numpy
                    nb_classes=vx,
                    nb_samples=self.nb_softmax_samples[0],
                    labels=x,
                    is_training=is_training,
                    freq=self.freq_x
                )
                feed_dict[self.model.support_x] = support_x
                feed_dict[self.model.importance_x] = importance_x
                #logging.info('X: probable=%s negative=%s', nb_probable_x, support_x.shape[0] - nb_probable_x)

            # Y side
            vy = self.training.vocab_size(1)
            if 0 < self.nb_softmax_samples[1] < vy and self.softmax_approximation[1] == 'botev-batch':  # in case we are employing a sampled softmax
                support_y, importance_y, nb_probable_y = np_get_support(  # we sample the support using numpy
                    nb_classes=vy,
                    nb_samples=self.nb_softmax_samples[1],
                    labels=y,
                    is_training=is_training,
                    freq=self.freq_y
                )
                feed_dict[self.model.support_y] = support_y
                feed_dict[self.model.importance_y] = importance_y
                #logging.info('Y: probable=%s negative=%s', nb_probable_y, support_y.shape[0] - nb_probable_y)

            yield feed_dict

    def checkpoint(self, step, training_loss):

        save_path = self.model.save(self.session, path="{}/model.latest.ckpt".format(self.output_dir))
        logging.info("Model saved in file: %s", save_path)

        self.tracker.register(  # track best training loss
            step=step,
            name='training.objective',
            value=training_loss,
            asc=False
        )

        # evaluate on development set
        if not self.val:
            return

        val_aer, val_acc_x, val_acc_y, val_loss, val_ce_x, val_ce_y, val_kl = self.evaluate(self.val,
                                                                                iter(self.val_wa))

        logging.info("Steps {} val_loss {:.4f} val_acc_x {:1.4f} val_acc_y {:1.4f} val_aer {:1.2f} ".format(
            step,
            val_loss,
            val_acc_x,
            val_acc_y,
            val_aer))

        # Save best models according to different validation metrics

        self.tracker.register(
            step=step,
            name='validation.objective',
            value=val_loss,
            asc=False
        )

        self.tracker.register(
            step=step,
            name='validation.aer',
            value=val_aer,
            asc=False
        )

        #self.tracker.register(
        #    step=step,
        #    name='validation.acc_x',
        #    value=val_acc_x,
        #    asc=True
        #)

        #self.tracker.register(
        #    step=step,
        #    name='validation.acc_y',
        #    value=val_acc_y,
        #    asc=True
        #)

    def train(self, ckpt_interval=1000):
        """Trains a model."""

        steps, epoch_steps = 0, 1
        loss = 0.0

        for epoch_id in range(1, self.num_epochs + 1):

            loss = 0.0
            ce_x = 0.0
            ce_y = 0.0
            kl = 0.0
            kl_s = 0.0
            kl_z = 0.0
            acc_x_correct = 0
            acc_x_total = 0
            acc_y_correct = 0
            acc_y_total = 0

            for epoch_steps, feed_dict in enumerate(self.get_generator(self.training, training_phase=True), 1):

                # things we want TF to return to us from the computation
                fetches = {
                    "optimizer_state": self.optimizer_state,
                    "loss": self.model.loss,
                    "ce_x": self.model.ce_x,
                    "ce_y": self.model.ce_y,
                    "kl_s": self.model.kl_s,
                    "kl_z": self.model.kl_z,
                    "kl": self.model.kl,
                    "acc_x_correct": self.model.accuracy_x_correct,
                    "acc_x_total": self.model.accuracy_x_total,
                    "acc_y_correct": self.model.accuracy_y_correct,
                    "acc_y_total": self.model.accuracy_y_total,
                    "pa_x": self.model.pa_x,
                    "px_z": self.model.px_z,
                    "py_xza": self.model.py_xza
                }

                res = self.session.run(fetches, feed_dict=feed_dict)

                loss += res["loss"]
                # breakdown of loss
                ce_x += res['ce_x']
                ce_y += res['ce_y']
                kl_s += res["kl_s"]
                kl_z += res["kl_z"]
                kl += res["kl"]
                # accuracy of next word prediction
                acc_x_correct += res["acc_x_correct"]
                acc_x_total += res["acc_x_total"]
                acc_y_correct += res["acc_y_correct"]
                acc_y_total += res["acc_y_total"]
                steps += 1

                # Log some running averages
                if epoch_steps % 100 == 0:
                    logging.info("Epoch {:3d} Steps {:5d} Iteration {:5d} loss {:.4f} ce_x {:.4f} ce_y {:.4f} kl {:.4f} kl_s {:.4f} kl_z {:.4f} alpha_s {:.4f} alpha_z {:.4f} accuracy_x {:1.4f} accuracy_y {:1.4f}".format(
                        epoch_id, steps, epoch_steps, loss / epoch_steps, ce_x / epoch_steps, ce_y / epoch_steps, kl / epoch_steps, kl_s / epoch_steps, kl_z / epoch_steps,
                        self.annealing_s.alpha(),
                        self.annealing_z.alpha(),
                        acc_x_correct / acc_x_total,
                        acc_y_correct / acc_y_total))

                # Possibly update annealing schedules after each parameter update
                self.annealing_s.update()
                self.annealing_z.update()

                if steps % ckpt_interval == 0:
                    #check point at every epoch
                    self.checkpoint(steps, loss / float(epoch_steps))

            # Log end of epoch
            logging.info("End of Epoch {:3d} Steps {:5d} loss {:.4f} ce_x {:.4f} ce_y {:.4f} kl {:.4f} kl_s {:.4f} kl_z {:.4f} alpha_s {:.4f} alpha_z {:.4f} accuracy_x {:1.4f} accuracy_y {:1.4f}".format(
                epoch_id, steps, loss / epoch_steps, ce_x / epoch_steps, ce_y / epoch_steps, kl / epoch_steps, kl_s / epoch_steps, kl_z / epoch_steps,
                self.annealing_s.alpha(),
                self.annealing_z.alpha(),
                acc_x_correct / acc_x_total,
                acc_y_correct / acc_y_total))

        # Final checkpoint
        self.checkpoint(steps, loss / float(epoch_steps))
        if self.val:  # compile a report
            with open('%s/report-best.txt' % self.output_dir, 'w') as fo:
                for metric, metric_data in self.tracker:
                    print('metric=%s value=%s step=%d path=%s' %
                          (metric,
                           metric_data.value,
                           metric_data.step,
                           metric_data.path), file=fo)

    def evaluate(self, corpus: Multitext, wa_iterator=None):
        """Evaluate the model on a data set."""
        metric = AERSufficientStatistics()
        accuracy_x_correct = 0
        accuracy_x_total = 0
        accuracy_y_correct = 0
        accuracy_y_total = 0
        loss = 0.0
        ce_x = 0.0
        ce_y = 0.0
        kl = 0.0

        for batch_id, feed_dict in enumerate(self.get_generator(corpus, training_phase=False), 1):

            # things we want TF to return to us from the computation
            fetches = {
                "optimizer_state": self.optimizer_state,
                "loss": self.model.loss,
                "ce_x": self.model.ce_x,
                "ce_y": self.model.ce_y,
                "kl": self.model.kl,
                "acc_x_correct": self.model.accuracy_x_correct,
                "acc_x_total": self.model.accuracy_x_total,
                "acc_y_correct": self.model.accuracy_y_correct,
                "acc_y_total": self.model.accuracy_y_total,
                "pa_x": self.model.pa_x,
                "px_z": self.model.px_z,
                "py_xza": self.model.py_xza
            }

            res = self.session.run(fetches, feed_dict=feed_dict)

            # Track some quantities
            accuracy_x_correct += res['acc_x_correct']
            accuracy_x_total += res['acc_x_total']
            accuracy_y_correct += res['acc_y_correct']
            accuracy_y_total += res['acc_y_total']
            loss += res['loss']
            ce_x += res['ce_x']
            ce_y += res['ce_y']
            kl += res['kl']

            # Get the input as it will be necessary for Viterbi
            x = feed_dict[self.model.x]
            y = feed_dict[self.model.y]
            align, prob = get_viterbi(x, y, res['pa_x'], res['py_xza'])

            if wa_iterator is not None:
                # Compute sufficient statistics for AER
                y_len = np.sum(np.sign(y), axis=1, dtype="int64")
                for alignment, N, (sure, probable) in zip(align, y_len, wa_iterator):
                    # the evaluation ignores NULL links, so we discard them
                    # j is 1-based in the naacl format
                    pred = set((aj, j) for j, aj in enumerate(alignment[:N], 1) if aj > 0)
                    metric.update(sure=sure, probable=probable, predicted=pred)

        accuracy_x = accuracy_x_correct / float(accuracy_x_total)
        accuracy_y = accuracy_y_correct / float(accuracy_y_total)

        return metric.aer(), accuracy_x, accuracy_y, loss / batch_id, ce_x / batch_id, ce_y / batch_id, kl / batch_id

def test_embedalign(  # input paths
        training_x, training_y,
        val_x, val_y, val_naacl,
        test_x, test_y, test_naacl,
        # output paths
        output_dir,
        ckpt_interval=1000,
        # data preprocessing
        nb_words=[1000, 1000],
        shortest_sequence=[1, 1],
        longest_sequence=[30, 30],
        bos_str=['-NULL-', None],
        eos_str=[None, None],
        lowercase=True,
        # architecture
        nb_softmax_samples=[0, 0],
        softmax_approximation=['botev-batch', 'botev-batch'],
        softmax_q_inv_temperature=[0., 0.],
        dx=128,
        dh=128,
        q=VariationalApproximationSpecs(dz=128),
        mc_kl=False,
        attention_z=False,
        improved_features_for_py=False,
        # optimiser
        nb_epochs=50,
        batch_size=100,
        optimizer=tf.train.RMSPropOptimizer(learning_rate=0.001),
        grad_clipping=None,
        annealing_s=AnnealingSchedule(),
        annealing_z=AnnealingSchedule(),
        config=None
):
    tks, training = prepare_training(training_x,
                                     training_y,
                                     nb_words=nb_words,
                                     shortest_sequence=shortest_sequence,
                                     longest_sequence=longest_sequence,
                                     bos_str=bos_str,
                                     eos_str=eos_str,
                                     lowercase=lowercase)
    logging.info("{}/tokenizer.pickle".format(output_dir))
    dill.dump(tks, open("{}/tokenizer.pickle".format(output_dir), 'wb'))

    if val_x:
        val, val_wa = prepare_validation(tks, val_x, val_y, val_naacl,
                                         shortest_sequence=shortest_sequence,
                                         longest_sequence=longest_sequence)

        test, test_wa = (None, None)#prepare_test(tks, test_x, test_y, test_naacl)
    else:
        val, val_wa = (None, None)
        test, test_wa = (None, None)

    tf.reset_default_graph()

    with tf.Session(config=config) as session:
        model = EmbedAlignModel(vx=training.vocab_size(0),
                                vy=training.vocab_size(1),
                                dx=dx, dh=dh,
                                q=q, mc_kl=mc_kl,
                                nb_softmax_samples=nb_softmax_samples,
                                softmax_approximation=softmax_approximation,
                                attention_z=attention_z,
                                improved_features_for_py=improved_features_for_py,
                                session=session)

        trainer = EmbedAlignTrainer(model,
                                    training,
                                    val, val_wa,
                                    test, test_wa,
                                    output_dir,
                                    # architecture
                                    nb_softmax_samples=nb_softmax_samples,
                                    softmax_approximation=softmax_approximation,
                                    softmax_q_inv_temperature=softmax_q_inv_temperature,
                                    # optimizer
                                    num_epochs=nb_epochs,
                                    batch_size=batch_size,
                                    optimizer=optimizer,
                                    grad_clipping=grad_clipping,
                                    annealing_s=annealing_s,
                                    annealing_z=annealing_z,
                                    session=session)

        # now first TF needs to initialize all the variables
        logging.info('Initializing variables...')
        session.run(tf.global_variables_initializer())

        logging.info('Training started')
        trainer.train(ckpt_interval=ckpt_interval)
        #TODO save model!


from typing import List
import time

class Solution:
    def arrayStringsAreEqual(self, word1: List[str], word2: List[str]) -> bool:
        def listToString(s):
            # initialize an empty string
            str1 = ""

            # traverse in the string
            for ele in s:
                str1 += ele

            # return string
            return str1

        word1Str = listToString(word1)
        word2Str = listToString(word2)
        if word1Str == word2Str:
            return True
        else:
            return False

# Leetcode Solution
# class Solution:
#     def arrayStringsAreEqual(self, word1: List[str], word2: List[str]) -> bool:
#         return True if ''.join(word1) == ''.join(word2) else False

start = time.time()
s = Solution()
word1 = ["ab", "c"]
word2 = ["a", "bc"]

# word1 = ["a", "cb"]
# word2 = ["ab", "c"]
# word1=["abc", "d", "defg"]
# word2 = ["abcddefg"]
returnval = s.arrayStringsAreEqual(word1, word2)
print(returnval)
end = time.time()
print("The time of execution of above program is :", (end - start) * 10**3, "ms")


from django.urls import path

from blogs import views

app_name = 'blogs'

urlpatterns = [
    path('', views.home_page),
    path('post/<slug:slug>', views.PostView.as_view(), name='post'),
    path('featured/', views.FeaturedListView.as_view(), name='featured'),
    path('category/<slug:slug>', views.CategoryListView.as_view(), name='category'),
    path('search/', views.SearchResultsView.as_view(), name='search'),
]


from easydict import EasyDict as edict

from mindspore.train.model import Model
from mindspore.train import amp
from mindspore.train.amp import _config_level
from mindspore.common import dtype as mstype
from mindspore.nn.wrap.cell_wrapper import _VirtualDatasetCell
from mindspore.parallel._utils import _get_parallel_mode
from mindspore.train.model import ParallelMode
from mindspore.train.amp import _do_keep_batchnorm_fp32
from mindspore.train.amp import _add_loss_network
from mindspore import context
from mindspore import nn
from mindspore import Tensor
from mindspore.ops import composite as C
from mindspore.ops import operations as P
from mindspore.ops import functional as F
from mindspore.ops.operations import NPUGetFloatStatus
from mindspore.ops.operations import NPUAllocFloatStatus
from mindspore.ops.operations import NPUClearFloatStatus
from mindspore.ops.operations import ReduceSum
from mindspore.ops.operations import LessEqual
from mindspore.parallel._utils import _get_gradients_mean
from mindspore.parallel._utils import _get_device_num
from mindspore.nn.wrap.grad_reducer import DistributedGradReducer
from mindspore.common.parameter import Parameter
from mindspore.nn.wrap.loss_scale import _grad_overflow
from mindspore.nn import Cell
from mindspore import ParameterTuple

from mindarmour.utils.logger import LogUtil
from mindarmour.utils import _check_param as validator
from mindarmour.utils._check_param import check_value_positive, check_param_type
from mindarmour.utils._check_param import check_int_positive, check, check_value_type
from ..mechanisms.mechanisms import _MechanismsParamsUpdater

LOGGER = LogUtil.get_instance()
TAG = 'DP model'

_grad_scale = C.MultitypeFuncGraph("grad_scale")
_reciprocal = P.Reciprocal()

@_grad_scale.register("Tensor", "Tensor")
def tensor_grad_scale(scale, grad):
    """ grad scaling """
    return grad * F.cast(_reciprocal(scale), F.dtype(grad))

class DPModel(Model):
    """
    DPModel is used for constructing a model for differential privacy training.
    This class is overload mindspore.train.Model.

    For details, please check `Protecting User Privacy with Differential Privacy Mechanism
    <https://mindspore.cn/mindarmour/docs/en/master/protect_user_privacy_with_differential_privacy.html#%E5%B7%AE%E5%88%86%E9%9A%90%E7%A7%81>`_.

    Args:
        micro_batches (int): The number of small batches split from an original
            batch. Default: ``2``.
        norm_bound (float): The norm bound that is used to clip the gradient of
         each sample. Default: ``1.0``.
        noise_mech (Mechanisms): The object can generate the different type of
            noise. Default: ``None``.
        clip_mech (Mechanisms): The object is used to update the adaptive clip.
            Default: ``None``.
        optimizer (Cell): Optimizer used for differential privacy training, which can be original mindspore
            optimizers (for example, Momentum optimizer) or optimizers generated by DPOptimizerClassFactory.
            Default: ``nn.Momentum``.

    Raises:
        ValueError: If `optimizer` is ``None``.
        ValueError: If `optimizer` is not DPOptimizer and `noise_mech` is ``None``.
        ValueError: If `optimizer` is DPOptimizer and `noise_mech` is not ``None``.
        ValueError: If `noise_mech` or DPOptimizer's mech method is adaptive while `clip_mech` is not ``None``.

    """

    def __init__(self, micro_batches=2, norm_bound=1.0, noise_mech=None,
                 clip_mech=None, optimizer=nn.Momentum, **kwargs):
        if micro_batches:
            self._micro_batches = check_int_positive('micro_batches',
                                                     micro_batches)
        else:
            self._micro_batches = None
        norm_bound = check_param_type('norm_bound', norm_bound, float)
        norm_bound = check_value_positive('norm_bound', norm_bound)
        norm_bound = Tensor(norm_bound, mstype.float32)
        self._norm_bound = Parameter(norm_bound, 'norm_bound')
        if optimizer is None:
            msg = 'Optimizer need to be set, but got None.'
            LOGGER.error(TAG, msg)
            raise ValueError(msg)
        opt_name = optimizer.__class__.__name__
        # Check whether noise_mech and DPOptimizer are both None or not None, if so, raise ValueError.
        # And check whether noise_mech or DPOtimizer's mech method is adaptive while clip_mech is not None,
        # if so, raise ValuerError too.
        if noise_mech is not None and "DPOptimizer" in opt_name:
            msg = 'DPOptimizer is not supported while noise_mech is not None'
            LOGGER.error(TAG, msg)
            raise ValueError(msg)
        if noise_mech is None:
            if "DPOptimizer" in opt_name:
                if 'Ada' in optimizer._mech.__class__.__name__ and clip_mech is not None:
                    msg = "When DPOptimizer's mech method is adaptive, clip_mech must be None."
                    LOGGER.error(TAG, msg)
                    raise ValueError(msg)
            else:
                msg = 'DPModel should set noise_mech or DPOptimizer configure, ' \
                      'please refer to example.'
                LOGGER.error(TAG, msg)
                raise ValueError(msg)
        self._noise_mech = noise_mech
        if noise_mech is not None:
            if 'Ada' in noise_mech.__class__.__name__ and clip_mech is not None:
                msg = 'When noise_mech is Adaptive, clip_mech must be None.'
                LOGGER.error(TAG, msg)
                raise ValueError(msg)

        if clip_mech is None or isinstance(clip_mech, Cell):
            self._clip_mech = clip_mech
        super(DPModel, self).__init__(optimizer=optimizer, **kwargs)

        # judge device_target, only GPU or Ascend is supported until now
        device = context.get_context("device_target")
        if device not in ["GPU", "Ascend"]:
            msg = "'device_target' for DP training should be 'GPU' or 'Ascend', but got {}.".format(device)
            LOGGER.error(TAG, msg)
            raise ValueError(msg)

    def _amp_build_train_network(self, network, optimizer, loss_fn=None,
                                 level='O0', **kwargs):
        """
        Build the mixed precision training cell automatically.

        Args:
            network (Cell): Definition of the network.
            loss_fn (Union[None, Cell]): Definition of the loss_fn. If None,
                the `network` should have the loss inside. Default: ``None``.
            optimizer (Optimizer): Optimizer to update the Parameter.
            level (str): Supports [O0, O2]. Default: "O0".

                - O0: Do not change.
                - O2: Cast network to float16, keep batchnorm and `loss_fn`
                  (if set) run in float32, using dynamic loss scale.

            cast_model_type (:class:`mindspore.dtype`): Supports `mstype.float16`
                or `mstype.float32`. If set to `mstype.float16`, use `float16`
                mode to train. If set, overwrite the level setting.
            keep_batchnorm_fp32 (bool): Keep Batchnorm run in `float32`. If set,
                overwrite the level setting.
            loss_scale_manager (Union[None, LossScaleManager]): If None, not
                scale the loss, or else scale the loss by LossScaleManager.
                If set, overwrite the level setting.
        """
        check_value_type('network', network, nn.Cell, None)
        check_value_type('optimizer', optimizer, nn.Optimizer, None)
        check('level', level, "", ['O0', 'O2'], validator.IN, None)
        self._check_kwargs(kwargs)
        config = dict(_config_level[level], **kwargs)
        config = edict(config)

        if config.cast_model_type == mstype.float16:
            network.to_float(mstype.float16)

            if config.keep_batchnorm_fp32:
                _do_keep_batchnorm_fp32(network)

        if loss_fn:
            network = _add_loss_network(network, loss_fn,
                                        config.cast_model_type)

        if _get_parallel_mode() in (
                ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL):
            network = _VirtualDatasetCell(network)

        loss_scale = 1.0
        if config.loss_scale_manager is not None:
            loss_scale_manager = config.loss_scale_manager
            loss_scale = loss_scale_manager.get_loss_scale()
            update_cell = loss_scale_manager.get_update_cell()
            if update_cell is not None:
                # only cpu not support `TrainOneStepWithLossScaleCell` for control flow.
                if not context.get_context("enable_ge") and context.get_context(
                        "device_target") == "CPU":
                    msg = "Only `loss_scale_manager=None` and " \
                          "`loss_scale_manager=FixedLossScaleManager(drop_overflow" \
                          "_update=False)` are supported in current version. " \
                          "If you use `O2` option, please use " \
                          "`loss_scale_manager=None` or `FixedLossScaleManager`"
                    LOGGER.error(TAG, msg)
                    raise ValueError(msg)
                network = _TrainOneStepWithLossScaleCell(network,
                                                         optimizer,
                                                         scale_update_cell=update_cell,
                                                         micro_batches=self._micro_batches,
                                                         norm_bound=self._norm_bound,
                                                         clip_mech=self._clip_mech,
                                                         noise_mech=self._noise_mech).set_train()
                return network

        network = _TrainOneStepCell(network,
                                    optimizer,
                                    self._norm_bound,
                                    loss_scale,
                                    micro_batches=self._micro_batches,
                                    clip_mech=self._clip_mech,
                                    noise_mech=self._noise_mech).set_train()
        return network

    def _build_train_network(self):
        """Build train network"""
        network = self._network
        if self._micro_batches:
            if self._optimizer:
                if self._loss_scale_manager_set:
                    network = self._amp_build_train_network(network,
                                                            self._optimizer,
                                                            self._loss_fn,
                                                            level=self._amp_level,
                                                            loss_scale_manager=self._loss_scale_manager,
                                                            keep_batchnorm_fp32=self._keep_bn_fp32)
                else:
                    network = self._amp_build_train_network(network,
                                                            self._optimizer,
                                                            self._loss_fn,
                                                            level=self._amp_level,
                                                            keep_batchnorm_fp32=self._keep_bn_fp32)
            elif self._loss_fn:
                network = nn.WithLossCell(network, self._loss_fn)
        else:
            if self._optimizer:
                if self._loss_scale_manager_set:
                    network = amp.build_train_network(network,
                                                      self._optimizer,
                                                      self._loss_fn,
                                                      level=self._amp_level,
                                                      loss_scale_manager=self._loss_scale_manager,
                                                      keep_batchnorm_fp32=self._keep_bn_fp32)
                else:
                    network = amp.build_train_network(network,
                                                      self._optimizer,
                                                      self._loss_fn,
                                                      level=self._amp_level,
                                                      keep_batchnorm_fp32=self._keep_bn_fp32)
            elif self._loss_fn:
                network = nn.WithLossCell(network, self._loss_fn)

        if self._parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL,
                                   ParallelMode.AUTO_PARALLEL):
            network.set_auto_parallel()
        return network

class _ClipGradients(nn.Cell):
    """
    Clip gradients.

    Inputs:
        grads (tuple[Tensor]): Gradients to clip.
        clip_norm (float): The l2-norm bound used to clip the gradients.
        cur_norm (float): The l2-norm of grads. If None, the norm will be
            calculated in this function. Default: ``None``.

    Outputs:
        tuple[Tensor], clipped gradients.
    """

    def __init__(self):
        super(_ClipGradients, self).__init__()
        self._add = P.Add()
        self._reduce_sum = P.ReduceSum()
        self._square_all = P.Square()
        self._sqrt = P.Sqrt()

    def construct(self, grads, clip_norm, cur_norm=None):
        """
        construct a compute flow.
        """
        if cur_norm is None:
            # calculate current l2-norm of grads
            square_sum = Tensor(0, mstype.float32)
            for grad in grads:
                square_sum = self._add(square_sum, self._reduce_sum(self._square_all(grad)))
            cur_norm = self._sqrt(square_sum)

        if cur_norm <= clip_norm:
            return grads

        new_grads = ()
        for grad in grads:
            clipped_grad = grad * (clip_norm / cur_norm)
            new_grads = new_grads + (clipped_grad,)
        return new_grads

class _TupleAdd(nn.Cell):
    def __init__(self):
        super(_TupleAdd, self).__init__()
        self.add = P.Add()
        self.hyper_map = C.HyperMap()

    def construct(self, input1, input2):
        """Add two tuple of data."""
        out = self.hyper_map(self.add, input1, input2)
        return out

class _TrainOneStepWithLossScaleCell(Cell):
    r"""
    Network training with loss scaling.

    This is a training step with loss scaling. It takes a network, an optimizer
    and possibly a scale update Cell as args. The loss scale value can be
    updated in both host side or device side. The TrainOneStepWithLossScaleCell
    will be compiled to be graph which takes `data`, `label`, `sens` as input
    data. The `sens` is acting as loss scaling value. If you want to update it
    on host side, the value should be provided. If `sens` is not given, the loss
    scale update logic should be provied by `scale_update_cell`. If
    `scale_update_cell` is not None and `sens` is provided, the
    `scale_update_cell` will be ignored.

    Args:
        network (Cell): The training network.
        optimizer (Cell): Optimizer for updating the weights.
        scale_update_cell(Cell): The loss scaling update logic cell.
            Default: ``None``.
        micro_batches (int): The number of small batches split from an original
            batch. Default: ``None``.
        norm_bound (Tensor): The norm bound that is used to clip the gradient of
         each sample. Default: 1.0.
        noise_mech (Mechanisms): The object can generate the different type of
            noise. Default: ``None``.

    Inputs:
        - **inputs** (Tensor) - Tensor of shape :math:`(N, \ldots)`.
        - **label** (Tensor) - Tensor of shape :math:`(N, \ldots)`.
        - **scaling_sens** (Tensor) - Tensor of shape :math:`()`.

    Outputs:
        Tuple of 3 Tensor, the loss, overflow flag and current loss scaling value.

        - **loss** (Tensor) -  Tensor with shape :math:`()`.
        - **overflow** (Tensor) -  Tensor with shape :math:`()`, type is bool.
        - **loss_scale** (Tensor) -  Tensor with shape :math:`()`.
    """

    def __init__(self, network, optimizer, scale_update_cell=None,
                 micro_batches=None, norm_bound=1.0, noise_mech=None,
                 clip_mech=None):
        super(_TrainOneStepWithLossScaleCell, self).__init__(auto_prefix=False)
        self.network = network
        self.network.set_grad()
        self.network.add_flags(defer_inline=True)
        self.weights = ParameterTuple(network.trainable_params())
        self.optimizer = optimizer
        self.grad = C.GradOperation(get_by_list=True, sens_param=True)
        self.hyper_map = C.HyperMap()
        if context.get_context("device_target") == "GPU":
            self.gpu_target = True
            self.float_status = P.FloatStatus()
            self.addn = P.AddN()
            self.reshape = P.Reshape()
        else:
            self.gpu_target = False
            self.alloc_status = NPUAllocFloatStatus()
            self.get_status = NPUGetFloatStatus()
            self.clear_status = NPUClearFloatStatus()
        self.reduce_sum = ReduceSum(keep_dims=False)
        self.base = Tensor(1, mstype.float32)
        self.less_equal = LessEqual()
        self.allreduce = P.AllReduce()
        self.parallel_mode = _get_parallel_mode()
        self.grad_reducer = F.identity
        self.reducer_flag = self.parallel_mode in [ParallelMode.DATA_PARALLEL,
                                                   ParallelMode.HYBRID_PARALLEL]
        if self.reducer_flag:
            mean = _get_gradients_mean()
            degree = _get_device_num()
            self.grad_reducer = DistributedGradReducer(optimizer.parameters,
                                                       mean, degree)
        self.is_distributed = self.parallel_mode != ParallelMode.STAND_ALONE

        self.loss_scale = None
        self.loss_scaling_manager = scale_update_cell
        if scale_update_cell:
            self.loss_scale = Parameter(Tensor(scale_update_cell.get_loss_scale(), dtype=mstype.float32),
                                        name="loss_scale")
        self.add_flags(has_effect=True)

        # dp params
        self._micro_batches = micro_batches
        self._norm_bound = norm_bound
        self._split = P.Split(0, self._micro_batches)
        self._clip_by_global_norm = _ClipGradients()
        self._noise_mech = noise_mech
        self._clip_mech = clip_mech
        self._add = P.Add()
        self._norm = nn.Norm()
        self._tuple_add = _TupleAdd()
        self._hyper_map = C.HyperMap()
        self._micro_float = Tensor(micro_batches, mstype.float32)
        self._zero = Tensor(0, mstype.float32)
        self._assign = P.Assign()
        self._div = P.Div()
        self._sqrt = P.Sqrt()
        self._reduce_sum = P.ReduceSum()
        self._square_all = P.Square()
        self._less = P.Less()
        self._cast = P.Cast()

        self._noise_mech_param_updater = None
        if self._noise_mech is not None and self._noise_mech._decay_policy is not None:
            self._noise_mech_param_updater = _MechanismsParamsUpdater(
                decay_policy=self._noise_mech._decay_policy,
                decay_rate=self._noise_mech._noise_decay_rate,
                cur_noise_multiplier=self._noise_mech._noise_multiplier,
                init_noise_multiplier=self._noise_mech._initial_noise_multiplier)

    def construct(self, data, label, sens=None):
        """
        construct a compute flow.
        """
        init = False
        if not self.gpu_target:
            # init overflow buffer
            init = self.alloc_status()
            # clear overflow buffer
            self.clear_status(init)

        if sens is None:
            scaling_sens = self.loss_scale
        else:
            scaling_sens = sens

        # DP clip
        weights = self.weights
        record_datas = self._split(data)
        record_labels = self._split(label)
        # first index
        loss = self.network(record_datas[0], record_labels[0])
        scaling_sens_filled = C.ones_like(loss) * F.cast(scaling_sens,
                                                         F.dtype(loss))
        record_grad = self.grad(self.network, weights)(record_datas[0],
                                                       record_labels[0],
                                                       scaling_sens_filled)

        beta = self._zero
        square_sum = self._zero
        for grad in record_grad:
            square_sum = self._add(square_sum,
                                   self._reduce_sum(self._square_all(grad)))
        norm_grad = self._sqrt(square_sum)
        beta = self._add(beta,
                         self._cast(self._less(norm_grad, self._norm_bound),
                                    mstype.float32))
        record_grad = self._clip_by_global_norm(record_grad,
                                                self._norm_bound, norm_grad)
        grads = record_grad
        total_loss = loss
        for i in range(1, self._micro_batches):
            loss = self.network(record_datas[i], record_labels[i])
            scaling_sens_filled = C.ones_like(loss) * F.cast(scaling_sens,
                                                             F.dtype(loss))
            record_grad = self.grad(self.network, weights)(record_datas[i],
                                                           record_labels[i],
                                                           scaling_sens_filled)

            square_sum = self._zero
            for grad in record_grad:
                square_sum = self._add(square_sum,
                                       self._reduce_sum(self._square_all(grad)))
            norm_grad = self._sqrt(square_sum)
            beta = self._add(beta,
                             self._cast(self._less(norm_grad, self._norm_bound),
                                        mstype.float32))

            record_grad = self._clip_by_global_norm(record_grad,
                                                    self._norm_bound, norm_grad)
            grads = self._tuple_add(grads, record_grad)
            total_loss = P.Add()(total_loss, loss)
        loss = P.Div()(total_loss, self._micro_float)
        beta = self._div(beta, self._micro_batches)

        if self._noise_mech is not None:
            grad_noise_tuple = ()
            for grad_item in grads:
                grad_noise = self._noise_mech(grad_item)
                grad_noise_tuple = grad_noise_tuple + (grad_noise,)
            grads = self._tuple_add(grads, grad_noise_tuple)
            grads = self._hyper_map(F.partial(_grad_scale, self._micro_float),
                                    grads)
            # update mech parameters

            if self._noise_mech_param_updater is not None:
                multiplier = self._noise_mech_param_updater()
                loss = F.depend(loss, multiplier)

        grads = self.hyper_map(F.partial(_grad_scale, scaling_sens), grads)
        # apply grad reducer on grads
        grads = self.grad_reducer(grads)
        # get the overflow buffer
        if not self.gpu_target:
            self.get_status(init)
            # sum overflow buffer elements, 0:not overflow , >0:overflow
            flag_sum = self.reduce_sum(init, (0,))
        else:
            flag_sum = self.hyper_map(F.partial(_grad_overflow), grads)
            flag_sum = self.addn(flag_sum)
            # convert flag_sum to scalar
            flag_sum = self.reshape(flag_sum, (()))
        if self.is_distributed:
            # sum overflow flag over devices
            flag_reduce = self.allreduce(flag_sum)
            cond = self.less_equal(self.base, flag_reduce)
        else:
            cond = self.less_equal(self.base, flag_sum)
        overflow = cond
        if sens is None:
            overflow = self.loss_scaling_manager(self.loss_scale, cond)
        # if there is no overflow, do optimize
        if overflow:
            opt = False
        else:
            opt = self.optimizer(grads)
        ret = (loss, cond, scaling_sens)

        if self._clip_mech is not None:
            next_norm_bound = self._clip_mech(beta, self._norm_bound)
            P.assign(self._norm_bound, next_norm_bound)
        return F.depend(ret, opt)

class _TrainOneStepCell(Cell):
    r"""
    Network training package class.

    Wraps the network with an optimizer. The resulting Cell be trained with
    input data and label. Backward graph will be created in the construct
    function to do parameter updating. Different parallel modes are available
    to run the training.

    Args:
        network (Cell): The training network.
        optimizer (Cell): Optimizer for updating the weights.
        sens (Number): The scaling number to be filled as the input of back
            propagation. Default value is 1.0.
        micro_batches (int): The number of small batches split from an original
            batch. Default: ``None``.
        norm_bound (Tensor): The norm bound that is used to clip the gradient of
         each sample. Default: 1.0.
        noise_mech (Mechanisms): The object can generate the different type
            of noise. Default: ``None``.
        clip_mech (Mechanisms): The object is used to update the adaptive clip.
            Default: ``None``.

    Inputs:
        - **data** (Tensor) - Tensor of shape :math:`(N, \ldots)`.
        - **label** (Tensor) - Tensor of shape :math:`(N, \ldots)`.

    Outputs:
        Tensor, a scalar Tensor with shape :math:`()`.
    """

    def __init__(self, network, optimizer, norm_bound=1.0, sens=1.0,
                 micro_batches=None,
                 noise_mech=None, clip_mech=None):
        super(_TrainOneStepCell, self).__init__(auto_prefix=False)
        self.network = network
        self.network.set_grad()
        self.network.add_flags(defer_inline=True)
        self.weights = optimizer.parameters
        self.optimizer = optimizer
        self.grad = C.GradOperation(get_by_list=True, sens_param=True)
        self.sens = sens
        self.reducer_flag = False
        self.grad_reducer = None
        parallel_mode = _get_parallel_mode()
        if parallel_mode in (
                ParallelMode.DATA_PARALLEL, ParallelMode.HYBRID_PARALLEL):
            self.reducer_flag = True
        if self.reducer_flag:
            mean = _get_gradients_mean()
            degree = _get_device_num()
            self.grad_reducer = DistributedGradReducer(optimizer.parameters,
                                                       mean, degree)

        # dp params
        if micro_batches is None:
            msg = 'micro_batches must give in differential privacy, but got value: {}'.format(
                micro_batches)
            LOGGER.error(TAG, msg)
            raise ValueError(msg)
        self._micro_batches = micro_batches
        self._norm_bound = norm_bound
        self._split = P.Split(0, self._micro_batches)
        self._clip_by_global_norm = _ClipGradients()
        self._noise_mech = noise_mech
        self._clip_mech = clip_mech
        self._tuple_add = _TupleAdd()
        self._add = P.Add()
        self._norm = nn.Norm()
        self._hyper_map = C.HyperMap()
        self._zero = Tensor(0, mstype.float32)
        self._assign = P.Assign()
        self._div = P.Div()
        self._sqrt = P.Sqrt()
        self._reduce_sum = P.ReduceSum()
        self._square_all = P.Square()
        self._less = P.Less()
        self._cast = P.Cast()

        self._micro_float = Tensor(micro_batches, mstype.float32)

        self._noise_mech_param_updater = None
        if self._noise_mech is not None and self._noise_mech._decay_policy is not None:
            self._noise_mech_param_updater = _MechanismsParamsUpdater(
                decay_policy=self._noise_mech._decay_policy,
                decay_rate=self._noise_mech._noise_decay_rate,
                cur_noise_multiplier=self._noise_mech._noise_multiplier,
                init_noise_multiplier=self._noise_mech._initial_noise_multiplier)

    def construct(self, data, label):
        """
        construct a compute flow.
        """
        weights = self.weights
        record_datas = self._split(data)
        record_labels = self._split(label)
        loss = self.network(record_datas[0], record_labels[0])
        sens = P.Fill()(P.DType()(loss), P.Shape()(loss), self.sens)
        record_grad = self.grad(self.network, weights)(record_datas[0],
                                                       record_labels[0], sens)

        # calcu norm_grad
        square_sum = self._zero
        for grad in record_grad:
            square_sum = self._add(square_sum, self._reduce_sum(self._square_all(grad)))
        norm_grad = self._sqrt(square_sum)

        # calcu beta
        beta = self._zero
        if self._clip_mech is not None:
            beta = self._add(beta,
                             self._cast(self._less(norm_grad, self._norm_bound),
                                        mstype.float32))

        record_grad = self._clip_by_global_norm(record_grad,
                                                self._norm_bound, norm_grad)
        grads = record_grad
        total_loss = loss
        for i in range(1, self._micro_batches):
            loss = self.network(record_datas[i], record_labels[i])
            sens = P.Fill()(P.DType()(loss), P.Shape()(loss), self.sens)
            record_grad = self.grad(self.network, weights)(record_datas[i],
                                                           record_labels[i],
                                                           sens)

            # calcu norm_grad
            square_sum = self._zero
            for grad in record_grad:
                square_sum = self._add(square_sum,
                                       self._reduce_sum(self._square_all(grad)))
            norm_grad = self._sqrt(square_sum)

            # calcu beta
            if self._clip_mech is not None:
                beta = self._add(beta,
                                 self._cast(self._less(norm_grad, self._norm_bound),
                                            mstype.float32))

            record_grad = self._clip_by_global_norm(record_grad,
                                                    self._norm_bound, norm_grad)
            grads = self._tuple_add(grads, record_grad)
            total_loss = P.Add()(total_loss, loss)
        loss = self._div(total_loss, self._micro_float)

        if self._noise_mech is not None:
            grad_noise_tuple = ()
            for grad_item in grads:
                grad_noise = self._noise_mech(grad_item)
                grad_noise_tuple = grad_noise_tuple + (grad_noise,)
            grads = self._tuple_add(grads, grad_noise_tuple)
            grads = self._hyper_map(F.partial(_grad_scale, self._micro_float),
                                    grads)
            # update mech parameters
            if self._noise_mech_param_updater is not None:
                multiplier = self._noise_mech_param_updater()
                loss = F.depend(loss, multiplier)

        if self.reducer_flag:
            # apply grad reducer on grads
            grads = self.grad_reducer(grads)

        if self._clip_mech is not None:
            beta = self._div(beta, self._micro_batches)
            next_norm_bound = self._clip_mech(beta, self._norm_bound)
            self._norm_bound = self._assign(self._norm_bound, next_norm_bound)
            loss = F.depend(loss, self._norm_bound)

        return F.depend(loss, self.optimizer(grads))


import cv2
import numpy as np
from matplotlib import pyplot as plt
import pandas as pd
import PIL.Image as pilimg

image = cv2.imread('../data/vision2/test_dirty_mnist_2nd/50001.png', cv2.IMREAD_GRAYSCALE)
pix = np.array(image)

# 이미지 전처리 -------------------------------------

#254보다 작고 0이아니면 0으로 만들어주기
x_df2 = np.where((pix <= 254) & (pix != 0), 0, pix)

# 이미지 팽창
x_df3 = cv2.dilate(x_df2, kernel=np.ones((2, 2), np.uint8), iterations=1)

# 블러 적용, 노이즈 제거
x_df4 = cv2.medianBlur(src=x_df3, ksize= 5)

# 이미지 리쉐잎 -------------------------------------
# 리쉐잎

# 이전 파일 것
canny = cv2.Canny(x_df4, 30, 70)

sobelx = cv2.Sobel(x_df4, cv2.CV_64F, 1, 0, ksize=3)
sobely = cv2.Sobel(x_df4, cv2.CV_64F, 0, 1, ksize=3)
laplacian = cv2.Laplacian(x_df4, cv2.CV_8U)

images = [canny, sobelx, sobely, laplacian]
titles = ['canny', 'sobelx', 'sobely', 'laplacian']

for i in range(4):
    plt.subplot(2, 2, i+1), plt.imshow(images[i]), plt.title(titles[i])
    plt.xticks([]), plt.yticks([])

plt.show()


from flask import Flask, jsonify, request
from constants import GEOCODING_API_ENDPOINT, FORECAST_API_ENDPOINT, API_KEY
from mock import DIRECT_GEOCODING, FORECAST
import requests

app = Flask(__name__)

@app.route('/city/coordinates/<city_name>', methods=['GET'])
def get_coordinates(city_name):
    url = f"{GEOCODING_API_ENDPOINT}?q={city_name}&appid={API_KEY}"
    response = requests.get(url)
    response = response.json()
    # response = DIRECT_GEOCODING

    if not response:
        return jsonify({})

    return jsonify(response[0])

@app.route('/city/forecast/<lat>/<lon>/<number_of_days>/<interval>', methods=['GET'])
def get_forecast(lat, lon, number_of_days, interval):
    number_of_days = int(number_of_days)
    interval = int(interval)
    url = f"{FORECAST_API_ENDPOINT}?lat={lat}&lon={lon}&appid={API_KEY}"
    response = requests.get(url)
    response = response.json()
    # response = FORECAST
    days_report = response['list']
    result = []
    limit = 5 if number_of_days == 1 else 99999

    if number_of_days == 1:
        result = days_report[:limit + 1]
    else:
        for index in range(len(days_report)):
            if index and index % interval == 0 or index == 1 or interval == 1:
                result.append(days_report[index])

        result = result[1:number_of_days + 1]

    return result

if __name__ == "__main__":
    app.run(debug=True)


import matplotlib.pyplot as plt

a = input("Escribe el numero de personas que usan vehiculo: \n")
A = int(a)

while A < 0:
    print("¡Ha escrito un número negativo! Inténtelo de nuevo")
    a = input("Escribe el numero de personas que usan vehiculo: \n")
    A = int(a)

b = input("Escribe el numero de personas que usan motocicleta: \n")
B = int(b)

while B < 0:
    print("¡Ha escrito un número negativo! Inténtelo de nuevo")
    b = input("Escribe el numero de personas que usan motocicletas: \n")
    B = int(b)

c = input("Escribe el numero de personas que usan bicicletas: \n")
C = int(c)

while C < 0:
    print("¡Ha escrito un número negativo! Inténtelo de nuevo")
    c = input("Escribe el numero de personas que usan bicicletas: \n")
    C = int(c)

d = input("Escribe el numero de personas que usan el metro: \n")
D = int(d)

while D < 0:
    print("¡Ha escrito un número negativo! Inténtelo de nuevo")
    d = input("Escribe el numero de personas que usan el metro: \n")
    D = int(d)

Dato1 = float(A/(A+B+C+D)*100)
Dato2 = float(B/(A+B+C+D)*100)
Dato3 = float(C/(A+B+C+D)*100)
Dato4 = float(D/(A+B+C+D)*100)
print(Dato1, Dato2, Dato3, Dato4)
medios_transporte= "Vehiculo", "Motocicleta", "Bicicleta", "Metro"
sizes= [Dato1, Dato2,Dato3, Dato4]

fig1, ax1 =plt.subplots()
ax1.pie(sizes, labels=medios_transporte, autopct='%1.1f%%', shadow=True, startangle=90)

ax1.axis('equal')

plt.title("Principal Medio de transporte")
plt.legend()
plt.savefig('Grafica_pastel.png')
plt.show()


#!/usr/bin/python3
"""Onion Omega2 port of RPi powerbank using bq25895
https://github.com/tjohn327/raspberry_pi_ups/blob/master/src/powerpi.py
"""
from OmegaExpansion import onionI2C
import time
import sys
import logging

print("Loaded I2C Libraries")

class PowerOmega:
    PORT = 0
    ADDRESS = 0x6A      #I2C address of the ups
    #Refer to http://www.ti.com/lit/ds/symlink/bq25895.pdf for register maps

    REG_WATCHDOG = 0x07
    BYTE_WATCHDOG_STOP =  0b10001101 #Stop Watchdog timer
    REG_SYSMIN = 0x03
    BYTE_SYSMIN = 0b00110000
    REG_ILIM = 0x00 #ILIM register

    ####Edit this section to suit your needs######
    BYTE_ILIM =  0b00000000 #Input limit disabled
    #BYTE_ILIM =  0b01101000 #2A input current limit
    #BYTE_ILIM =  0b01111111 #3.25A input current limit
    BYTE_ICHG =  0b00001000 #.5A charging current limit
    #BYTE_ICHG =  0b00010000 #1A charging current limit
    BAT_CAPACITY = 1800 #Battery capacity in mAh
    CURRENT_DRAW = 2000 #Current draw in mAh approximately
    VBAT_LOW = 3.2
    ###############################################

    REG_ICHG = 0x04
    REG_ICHGR = 0x12
    REG_CONV_ADC = 0x02
    REG_BATFET = 0x09
    BYTE_BATFET = 0b01001000 #delay before battery is disconnected

    REG_CONV_ADC = 0x02
    BYTE_CONV_ADC_START = 0b10011101
    BYTE_CONV_ADC_STOP = 0b00011101
    REG_BATFET_DIS = 0x09
    BYTE_BATFET_DIS = 0b01101000
    REG_STATUS = 0x0B #address of status register
    REG_VBAT = 0x0E
    REG_FAULT = 0x0C
    REG_IBAT = 0x12
    REG_VBUS = 0x11
    VBAT_MAX = 4.208

    def __init__(self):
        pass

    def initialize(self):
        try:
            self.bus = onionI2C.OnionI2C(self.PORT)
            self.bus.writeByte(self.ADDRESS, self.REG_WATCHDOG, self.BYTE_WATCHDOG_STOP)
            self.bus.writeByte(self.ADDRESS, self.REG_ILIM,self.BYTE_ILIM)
            self.bus.writeByte(self.ADDRESS, self.REG_ICHG, self.BYTE_ICHG)
            self.bus.writeByte(self.ADDRESS, self.REG_BATFET, self.BYTE_BATFET)
            self.bus.writeByte(self.ADDRESS, self.REG_SYSMIN, self.BYTE_SYSMIN)
            logging.info("UPS initialized")
            return 0
        except Exception as ex:
            logging.error("Initialization failed, check connection to the UPS:"+ str(ex))
            return 1

    def _int_to_bool_list(self,num):
        return [bool(num & (1<<n)) for n in range(8)]

    def _vbat_convert(self,vbat_byte):
        vbat_bool = self._int_to_bool_list(vbat_byte)
        vbat = 2.304
        vbat += vbat_bool[6] * 1.280
        vbat += vbat_bool[5] * 0.640
        vbat += vbat_bool[4] * 0.320
        vbat += vbat_bool[3] * 0.160
        vbat += vbat_bool[2] * 0.08
        vbat += vbat_bool[1] * 0.04
        vbat += vbat_bool[0] * 0.02
        return vbat

    def _ibat_convert(self,ibat_byte):
        ibat_bool = self._int_to_bool_list(ibat_byte)
        ibat = 0
        ibat += ibat_bool[6] * 3200
        ibat += ibat_bool[5] * 1600
        ibat += ibat_bool[4] * 800
        ibat += ibat_bool[3] * 400
        ibat += ibat_bool[2] * 200
        ibat += ibat_bool[1] * 100
        ibat += ibat_bool[0] * 50
        return ibat

    def _vbus_convert(self,vbus_byte):
        vbus_bool = self._int_to_bool_list(vbus_byte)
        vbus = 2.6
        vbus += vbus_bool[6] * 6.4
        vbus += vbus_bool[5] * 3.2
        vbus += vbus_bool[4] * 1.6
        vbus += vbus_bool[3] * 0.8
        vbus += vbus_bool[2] * 0.4
        vbus += vbus_bool[1] * 0.2
        vbus += vbus_bool[0] * 0.1
        return vbus

    def _calc_bat_charge_percent(self,vbat):
        bat_charge_percent = (vbat-self.VBAT_LOW)/(self.VBAT_MAX - self.VBAT_LOW)
        if bat_charge_percent < 0:
            bat_charge_percent = 0
        elif bat_charge_percent > 1:
            bat_charge_percent = 1
        return bat_charge_percent

    def _calc_time_left(self,vbat):
        time_left = int(self._calc_bat_charge_percent(vbat) * 60 * self.BAT_CAPACITY / self.CURRENT_DRAW)
        if time_left < 0:
            time_left = 0
        return time_left

    def read_status(self, clear_fault=False):
        try:
            if clear_fault:
                self.bus.readBytes(self.ADDRESS, self.REG_FAULT, 1)
            self.bus.writeByte(self.ADDRESS, self.REG_CONV_ADC, self.BYTE_CONV_ADC_START)
            time.sleep(2)
            bat_fet = self.bus.readBytes(self.ADDRESS, self.REG_BATFET,1)[0]
            status = self.bus.readBytes(self.ADDRESS, self.REG_STATUS,1)[0]
            status = self._int_to_bool_list(int(status))
            vbat = self._vbat_convert(self.bus.readBytes(self.ADDRESS, self.REG_VBAT, 1)[0])
            ibat = self._ibat_convert(self.bus.readBytes(self.ADDRESS, self.REG_ICHGR, 1)[0])
            vbus = self._vbus_convert(self.bus.readBytes(self.ADDRESS, self.REG_VBUS, 1)[0])
            self.bus.writeByte(self.ADDRESS, self.REG_CONV_ADC, self.BYTE_CONV_ADC_STOP)
        except Exception as ex:
            logging.error("An exception occurred while reading values from the UPS: " + str(ex))
            time.sleep(2)
            return 1, None

        if status[2]:
            power_status = "Connected"
            time_left = -1
        else:
            power_status = "Not Connected"
            time_left = self._calc_time_left(vbat)

        if status[3] and status[4]:
            charge_status = "Charging done"
        elif status[4] and  not status[3]:
            charge_status = "Charging"
        elif not status[4] and status[3]:
            charge_status = "Pre-Charge"
        else:
            charge_status = "Not Charging"

        data = {
            'PowerInputStatus': power_status,
            'InputVoltage' : vbus,
            'ChargeStatus' : charge_status,
            'BatteryVoltage' : vbat,
            "BatteryPercentage" : int(self._calc_bat_charge_percent(vbat)*100),
            'ChargeCurrent' : ibat,
            'TimeRemaining' : int(time_left)
        }

        return 0, data

    def bat_disconnect(self):
        for i in (0,3):
            try:
                self.bus.writeByte(self.ADDRESS, self.REG_BATFET_DIS, self.BYTE_BATFET_DIS)
                return 0
            except:
                time.sleep(1)
        return 1

if __name__=="__main__":
    ups = PowerOmega()
    # Initial Charge start
    ups.initialize()
    print(ups.read_status(True))

    if len(sys.argv) > 1 and sys.argv[1] == "cycle":
        # Cycle BATFET to enable charge start
        # If powered from battery only this will switch off the device
        ups.bat_disconnect()
        time.sleep(10)
        print(ups.read_status(True))
        time.sleep(10)
        # Re-initialize and see if charge starts
        ups.initialize()
        time.sleep(10)
        print(ups.read_status(True))