import numpy as np
import cv2
from PIL import Image
#В этом скрипте делаем маску
#С помощью манипуляций с opencv создаем два файла
#первый - с контрастными крышами, второй - с выделенными дорогами
#затем - используя второй файл, убираем дороги с первого
image = "ZRYNEEUSVQ213QTY.png"

input = cv2.imread(image)

_, th = cv2.threshold(input, 130, 160, cv2.THRESH_TOZERO)

cv2.imwrite("th.png", th)

image = "th.png"
img = cv2.imread(image)

lower = np.array([0, 1, 1])
upper = np.array([0, 255, 255])
mask = cv2.inRange(img, lower, upper)
roads = cv2.bitwise_and(img, img, mask=mask)

height = int(np.size(img, 0))
width = int(np.size(img, 1))

for h in range(1, 500):
    for w in range(1, 500):
        color = str(roads[w, h])
        if color != "[0, 145, 153]":
            pass
        else:
            roads[h, w] = [0, 0, 255]

cv2.imwrite("roads.png", roads)

mask = Image.open("roads.png")
mask1 = Image.open("th.png")

for i in range (1, 6528):
    for j in range (1, 7733):
        if mask1.getpixel((i,j))[0] != 0 or mask1.getpixel((i,j))[1] != 0 or \
                mask1.getpixel((i,j))[2] != 0:
            mask1.putpixel((i, j), (255, 255, 255))
        if (mask.getpixel((i, j))[0] != 0) or (mask.getpixel((i, j))[1] != 0) or (mask.getpixel((i, j))[2] != 0):
            mask1.putpixel((i, j), (0, 0, 0))
mask1.save("dengi3.jpg", "JPEG")


from collections import defaultdict

class Solution:
    def valid_tree(self, n: int, edges: list[list[int]]) -> bool:
        seen: set[int] = set()
        children: dict[int, list[int]] = defaultdict(list)

        for x, y in edges:
            children[x].append(y)
            children[y].append(x)

        def dfs(node: int, prev: int) -> bool:
            if node in seen:
                return False

            seen.add(node)

            for child in children[node]:
                if child == prev:
                    continue
                if not dfs(child, node):
                    return False

            return True

        return dfs(0, 0) and len(seen) == n


from django.utils.module_loading import import_string
from django.urls import (RegexURLResolver, RegexURLPattern)
from CRM import settings
from collections import OrderedDict

def recursion_urls(pre_namespace, pre_url, urlpatterns, url_ordered_dict):
    # None, '/', urlpatterns, url_ordered_dict
    '''
    第一次递归：
        url(r'^', include('web.urls')),
        url(r'^rbac/', include('rbac.urls', namespace='rbac')),
    第二次递归：
        'rbac','/^rbac/', rbac.urls文件下的urlpatterns变量, url_ordered_dict
    '''
    for url in urlpatterns:
        if isinstance(url, RegexURLResolver):
            if pre_namespace:
                if url.namespace:
                    namespace = '%s:%s' % (pre_namespace, url.namespace)
                else:
                    namespace = pre_namespace
            else:
                if url.namespace:
                    namespace = url.namespace  # 'rbac'
                else:
                    namespace = None
            recursion_urls(namespace, pre_url + url.regex.pattern, url.url_patterns, url_ordered_dict)

        else:
            if pre_namespace:
                name = '%s:%s' % (pre_namespace, url.name)  # rbac:role_list
            else:
                name = url.name
            if not url.name:
                raise Exception('URL路由中必须设置name属性')
            url = pre_url + url._regex  # /^^login/
            url_ordered_dict[name] = {
                'url_name': name, 'url': url.replace('^', '').replace('$', '')
            }  # {'login':{'url_name': name, 'url': /login/},}

def all_url(ignore_namespace_list=None):
    ignore_list = ignore_namespace_list or []
    # 存放项目所有URL的有序字典
    url_ordered_dict = OrderedDict()
    # 获取项目的所以URL
    urls = import_string(settings.ROOT_URLCONF)
    urlpatterns = []
    '''
    # urlpatterns = [
    #     url(r'^', include('web.urls')),
    #     url(r'^rbac/', include('rbac.urls', namespace='rbac')),
    # ]
    '''
    for url in urls.urlpatterns:
        if isinstance(url, RegexURLResolver) and url.namespace in ignore_list:
            continue
        urlpatterns.append(url)
    recursion_urls(None, '/', urlpatterns, url_ordered_dict)
    return url_ordered_dict


from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.common.exceptions import NoSuchElementException
import time

driver = webdriver.Firefox()

driver.get('https://github.com/Vidoosh/Image-colorizer')
time.sleep(2)

code = driver.find_element(By.CSS_SELECTOR, '#repo-content-pjax-container > div > div > div.Layout.Layout--flowRow-until-md.Layout--sidebarPosition-end.Layout--sidebarPosition-flowRow-end > div.Layout-main > div.file-navigation.mb-3.d-flex.flex-items-start > span.d-none.d-md-flex.ml-2 > get-repo > feature-callout')
code.click()
time.sleep(1)
try:
    download = driver.find_element(By.CSS_SELECTOR, '#local-panel > ul > li:nth-child(3) > a')
    download.click()
    print("Download initiated successfully")

except NoSuchElementException as e:
    print("Download button not found")

finally:
    driver.quit()


# The following code was adapted from Week 3 Programming Assignment 2 in the Convolutional Neural Networks course by DeepLearning.AI offered on Coursera
# https://www.coursera.org/learn/convolutional-neural-networks/home/week/3

import tensorflow as tf
import numpy as np

from tensorflow.keras.layers import Input
from tensorflow.keras.layers import Conv2D
from tensorflow.keras.layers import MaxPooling2D
from tensorflow.keras.layers import Dropout
from tensorflow.keras.layers import Conv2DTranspose
from tensorflow.keras.layers import concatenate

import os
import numpy as np
import pandas as pd
import imageio
import matplotlib.pyplot as plt
get_ipython().run_line_magic('matplotlib', 'inline')

# load and split data
path = ''
image_path = os.path.join(path, './data/CameraRGB/')
mask_path = os.path.join(path, './data/CameraMask/')
image_list = os.listdir(image_path)
mask_list = os.listdir(mask_path)
image_list = [image_path+i for i in image_list]
mask_list = [mask_path+i for i in mask_list]

image_list_ds = tf.data.Dataset.list_files(image_list, shuffle=False)
mask_list_ds = tf.data.Dataset.list_files(mask_list, shuffle=False)

image_filenames = tf.constant(image_list)
masks_filenames = tf.constant(mask_list)

dataset = tf.data.Dataset.from_tensor_slices((image_filenames, masks_filenames))

# preprocess data
def process_path(image_path, mask_path):

    img = tf.io.read_file(image_path)
    img = tf.image.decode_png(img, channels=3)
    img = tf.image.convert_image_dtype(img, tf.float32)

    mask = tf.io.read_file(mask_path)
    mask = tf.image.decode_png(mask, channels=3)
    mask = tf.math.reduce_max(mask, axis=-1, keepdims=True)
    return img, mask

def preprocess(image, mask):
    input_image = tf.image.resize(image, (96, 128), method='nearest')
    input_mask = tf.image.resize(mask, (96, 128), method='nearest')

    return input_image, input_mask

image_ds = dataset.map(process_path)
processed_image_ds = image_ds.map(preprocess)

"""
This function implements the convolutional decoder block
"""
def upsampling_block(expansive_input, contractive_input, n_filters=32):

    # expansive_input is the input from the previous layer
    # contractive_input is the input from the previous skip layer

    up = Conv2DTranspose(n_filters, 3, strides=2, padding='same')(expansive_input)

    # merge the previous output and the contractive_input
    merge = concatenate([up, contractive_input], axis=3)
    conv = Conv2D(n_filters, 3, activation='relu', padding='same', kernel_initializer='HeNormal')(merge)
    conv = Conv2D(n_filters, 3, activation='relu', padding='same', kernel_initializer='HeNormal')(conv)

    return conv

"""
This function implements the U-Net model by combining the encoder and decoder paths
"""
def unet_model(input_size=(96, 128, 3), n_filters=32, n_classes=23):

    # input_size is the shape of the input
    # n_filters is the number of filters for convolutional layers
    # n_classes is the number of output classes
    inputs = Input(input_size)

    # encoding path
    cblock1 = conv_block(inputs, n_filters)

    # the first element of the output of each block will be the input of the next block
    # the number of filters at each new step doubles
    cblock2 = conv_block(cblock1[0], n_filters*2)
    cblock3 = conv_block(cblock2[0], n_filters*4)
    cblock4 = conv_block(cblock3[0], n_filters*8, dropout_prob=0.3)
    cblock5 = conv_block(cblock4[0], n_filters*16, dropout_prob=0.3, max_pooling=False)

    # decoding path
    ublock6 = upsampling_block(cblock5[0], cblock4[1],  n_filters*8)

    # the output of the previous block is the expansive_input and the corresponding encoder output skip connection is the contractive_input
    # the number of filters at each new step halves
    ublock7 = upsampling_block(ublock6, cblock3[1],  n_filters*4)
    ublock8 = upsampling_block(ublock7, cblock2[1],  n_filters*2)
    ublock9 = upsampling_block(ublock8, cblock1[1],  n_filters)

    conv9 = Conv2D(n_filters, 3, activation='relu', padding='same', kernel_initializer='he_normal')(ublock9)
    conv10 = Conv2D(n_classes, 1, padding='same')(conv9)

    model = tf.keras.Model(inputs=inputs, outputs=conv10)

    return model

# set model dimensions
img_height = 96
img_width = 128
num_channels = 3

unet = unet_model((img_height, img_width, num_channels))

unet.compile(optimizer='adam',
              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
              metrics=['accuracy'])

"""
This function displays both the input image and the corresponding true mask (desired output)
"""
def display(display_list):
    plt.figure(figsize=(15, 15))

    title = ['Input Image', 'True Mask', 'Predicted Mask']

    for i in range(len(display_list)):
        plt.subplot(1, len(display_list), i+1)
        plt.title(title[i])
        plt.imshow(tf.keras.preprocessing.image.array_to_img(display_list[i]))
        plt.axis('off')
    plt.show()

# create prediction
def create_mask(pred_mask):
    pred_mask = tf.argmax(pred_mask, axis=-1)
    pred_mask = pred_mask[..., tf.newaxis]
    return pred_mask[0]

# train the model
EPOCHS = 40
VAL_SUBSPLITS = 5
BUFFER_SIZE = 500
BATCH_SIZE = 32
processed_image_ds.batch(BATCH_SIZE)
train_dataset = processed_image_ds.cache().shuffle(BUFFER_SIZE).batch(BATCH_SIZE)
print(processed_image_ds.element_spec)
model_history = unet.fit(train_dataset, epochs=EPOCHS)

"""
This function displays the input, prediction, and true images
"""
def show_predictions(dataset=None, num=1):
    """
    Displays the first image of each of the num batches
    """
    if dataset:
        for image, mask in dataset.take(num):
            pred_mask = unet.predict(image)
            display([image[0], mask[0], create_mask(pred_mask)])
    else:
        display([sample_image, sample_mask,
             create_mask(unet.predict(sample_image[tf.newaxis, ...]))])

# show results
show_predictions(train_dataset, 6)

# References
# [1] https://www.coursera.org/learn/convolutional-neural-networks/programming/omqTR/image-segmentation-with-u-net


import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np

eps = 1e-7

class SCELoss(nn.Module):
    def __init__(self, num_classes=10, a=1, b=1):
        super(SCELoss, self).__init__()
        self.num_classes = num_classes
        self.a = a
        self.b = b
        self.cross_entropy = nn.CrossEntropyLoss()

    def forward(self, pred, labels):
        ce = self.cross_entropy(pred, labels)
        # RCE
        pred = F.softmax(pred, dim=1)
        pred = torch.clamp(pred, min=eps, max=1.0)
        label_one_hot = F.one_hot(labels, self.num_classes).float().to(pred.device)
        label_one_hot = torch.clamp(label_one_hot, min=1e-4, max=1.0)
        rce = (-1 * torch.sum(pred * torch.log(label_one_hot), dim=1))

        loss = self.a * ce + self.b * rce.mean()
        return loss

class RCELoss(nn.Module):
    def __init__(self, num_classes=10, scale=1.0):
        super(RCELoss, self).__init__()
        self.num_classes = num_classes
        self.scale = scale

    def forward(self, pred, labels):
        pred = F.softmax(pred, dim=1)
        pred = torch.clamp(pred, min=eps, max=1.0)
        label_one_hot = F.one_hot(labels, self.num_classes).float().to(pred.device)
        label_one_hot = torch.clamp(label_one_hot, min=1e-4, max=1.0)
        loss = (-1 * torch.sum(pred * torch.log(label_one_hot), dim=1))
        return self.scale * loss.mean()

class NRCELoss(nn.Module):
    def __init__(self, num_classes, scale=1.0):
        super(NRCELoss, self).__init__()
        self.num_classes = num_classes
        self.scale = scale

    def forward(self, pred, labels):
        pred = F.softmax(pred, dim=1)
        pred = torch.clamp(pred, min=eps, max=1.0)
        label_one_hot = F.one_hot(labels, self.num_classes).float().to(pred.device)
        label_one_hot = torch.clamp(label_one_hot, min=1e-4, max=1.0)
        norm = 1 / 4 * (self.num_classes - 1)
        rce = (-1 * torch.sum(pred * torch.log(label_one_hot), dim=1))
        return self.scale * norm * rce.mean()

class NCELoss(nn.Module):
    def __init__(self, num_classes, scale=1.0):
        super(NCELoss, self).__init__()
        self.num_classes = num_classes
        self.scale = scale

    def forward(self, pred, labels):
        pred = F.log_softmax(pred, dim=1)
        label_one_hot = F.one_hot(labels, self.num_classes).float().to(pred.device)
        loss = -1 * torch.sum(label_one_hot * pred, dim=1) / (-pred.sum(dim=1))
        return self.scale * loss.mean()

class MAELoss(nn.Module):
    def __init__(self, num_classes=10, scale=2.0):
        super(MAELoss, self).__init__()
        self.num_classes = num_classes
        self.scale = scale

    def forward(self, pred, labels):
        pred = F.softmax(pred, dim=1)
        label_one_hot = F.one_hot(labels, self.num_classes).float().to(pred.device)
        loss = 1. - torch.sum(label_one_hot * pred, dim=1)
        return self.scale * loss.mean()

class NMAE(nn.Module):
    def __init__(self, num_classes=10, scale=1.0):
        super(NMAE, self).__init__()
        self.num_classes = num_classes
        self.scale = scale

    def forward(self, pred, labels):
        pred = F.softmax(pred, dim=1)
        label_one_hot = F.one_hot(labels, self.num_classes).float().to(pred.device)
        norm = 1 / (self.num_classes - 1)
        loss = 1. - torch.sum(label_one_hot * pred, dim=1)
        return self.scale * norm * loss.mean()

class GCELoss(nn.Module):
    def __init__(self, num_classes=10, q=0.7):
        super(GCELoss, self).__init__()
        self.q = q
        self.num_classes = num_classes

    def forward(self, pred, labels):
        pred = F.softmax(pred, dim=1)
        pred = torch.clamp(pred, min=eps, max=1.0)
        label_one_hot = F.one_hot(labels, self.num_classes).float().to(pred.device)
        loss = (1. - torch.pow(torch.sum(label_one_hot * pred, dim=1), self.q)) / self.q
        return loss.mean()

class NGCELoss(nn.Module):
    def __init__(self, num_classes=10, q=0.7, scale=1.0):
        super(NGCELoss, self).__init__()
        self.num_classes = num_classes
        self.q = q
        self.scale = scale

    def forward(self, pred, labels):
        pred = F.softmax(pred, dim=1)
        pred = torch.clamp(pred, min=eps, max=1.0)
        label_one_hot = F.one_hot(labels, self.num_classes).float().to(pred.device)
        numerators = 1. - torch.pow(torch.sum(label_one_hot * pred, dim=1), self.q)
        denominators = self.num_classes - pred.pow(self.q).sum(dim=1)
        loss = numerators / denominators
        return self.scale * loss.mean()

class NCEandRCE(nn.Module):
    def __init__(self, alpha=1., beta=1., num_classes=10):
        super(NCEandRCE, self).__init__()
        self.num_classes = num_classes
        self.nce = NCELoss(num_classes=num_classes, scale=alpha)
        self.rce = RCELoss(num_classes=num_classes, scale=beta)

    def forward(self, pred, labels):
        return self.nce(pred, labels) + self.rce(pred, labels)

class NCEandMAE(nn.Module):
    def __init__(self, alpha=1., beta=1., num_classes=10):
        super(NCEandMAE, self).__init__()
        self.num_classes = num_classes
        self.nce = NCELoss(num_classes=num_classes, scale=alpha)
        self.mae = MAELoss(num_classes=num_classes, scale=beta)

    def forward(self, pred, labels):
        return self.nce(pred, labels) + self.mae(pred, labels)

class NLNL(torch.nn.Module):
    def __init__(self, train_loader, num_classes=10, ln_neg=1):
        super(NLNL, self).__init__()
        self.num_classes = num_classes
        self.ln_neg = ln_neg
        weight = torch.FloatTensor(num_classes).zero_() + 1.
        if not hasattr(train_loader.dataset, 'targets'):
            weight = [1] * num_classes
            weight = torch.FloatTensor(weight)
        else:
            for i in range(num_classes):
                weight[i] = (torch.from_numpy(np.array(train_loader.dataset.targets)) == i).sum()
            weight = 1 / (weight / weight.max())
        self.weight = weight.cuda()
        self.criterion = torch.nn.CrossEntropyLoss(weight=self.weight)
        self.criterion_nll = torch.nn.NLLLoss()

    def forward(self, pred, labels):
        labels_neg = (labels.unsqueeze(-1).repeat(1, self.ln_neg)
                      + torch.LongTensor(len(labels), self.ln_neg).cuda().random_(1, self.num_classes)) % self.num_classes
        labels_neg = torch.autograd.Variable(labels_neg)

        assert labels_neg.max() <= self.num_classes-1
        assert labels_neg.min() >= 0
        assert (labels_neg != labels.unsqueeze(-1).repeat(1, self.ln_neg)).sum() == len(labels)*self.ln_neg

        s_neg = torch.log(torch.clamp(1. - F.softmax(pred, 1), min=1e-5, max=1.))
        s_neg *= self.weight[labels].unsqueeze(-1).expand(s_neg.size()).cuda()
        labels = labels * 0 - 100
        loss = self.criterion(pred, labels) * float((labels >= 0).sum())
        loss_neg = self.criterion_nll(s_neg.repeat(self.ln_neg, 1), labels_neg.t().contiguous().view(-1)) * float((labels_neg >= 0).sum())
        loss = ((loss+loss_neg) / (float((labels >= 0).sum())+float((labels_neg[:, 0] >= 0).sum())))
        return loss

class FocalLoss(torch.nn.Module):
    '''
        https://github.com/clcarwin/focal_loss_pytorch/blob/master/focalloss.py
    '''

    def __init__(self, gamma=0.5, alpha=None, size_average=True):
        super(FocalLoss, self).__init__()
        self.gamma = gamma
        self.alpha = alpha
        if isinstance(alpha, (float, int)):
            self.alpha = torch.Tensor([alpha, 1-alpha])
        if isinstance(alpha, list):
            self.alpha = torch.Tensor(alpha)
        self.size_average = size_average

    def forward(self, input, target):
        if input.dim() > 2:
            input = input.view(input.size(0), input.size(1), -1)  # N,C,H,W => N,C,H*W
            input = input.transpose(1, 2)                         # N,C,H*W => N,H*W,C
            input = input.contiguous().view(-1, input.size(2))    # N,H*W,C => N*H*W,C
        target = target.view(-1, 1)

        logpt = F.log_softmax(input, dim=1)
        logpt = logpt.gather(1, target)
        logpt = logpt.view(-1)
        pt = torch.autograd.Variable(logpt.data.exp())

        if self.alpha is not None:
            if self.alpha.type() != input.data.type():
                self.alpha = self.alpha.type_as(input.data)
            at = self.alpha.gather(0, target.data.view(-1))
            logpt = logpt * torch.autograd.Variable(at)

        loss = -1 * (1-pt)**self.gamma * logpt
        if self.size_average:
            return loss.mean()
        else:
            return loss.sum()

class NormalizedFocalLoss(torch.nn.Module):
    def __init__(self, gamma=0.5, num_classes=10, alpha=None, size_average=True, scale=1.0):
        super(NormalizedFocalLoss, self).__init__()
        self.gamma = gamma
        self.size_average = size_average
        self.num_classes = num_classes
        self.scale = scale

    def forward(self, input, target):
        target = target.view(-1, 1)
        logpt = F.log_softmax(input, dim=1)
        normalizor = torch.sum(-1 * (1 - logpt.data.exp()) ** self.gamma * logpt, dim=1)
        logpt = logpt.gather(1, target)
        logpt = logpt.view(-1)
        pt = torch.autograd.Variable(logpt.data.exp())
        loss = -1 * (1-pt)**self.gamma * logpt
        loss = self.scale * loss / normalizor

        if self.size_average:
            return loss.mean()
        else:
            return loss.sum()

class NFLandRCE(torch.nn.Module):
    def __init__(self, alpha=1., beta=1., num_classes=10, gamma=0.5):
        super(NFLandRCE, self).__init__()
        self.num_classes = num_classes
        self.nfl = NormalizedFocalLoss(gamma=gamma, num_classes=num_classes, scale=alpha)
        self.rce = RCELoss(num_classes=num_classes, scale=beta)

    def forward(self, pred, labels):
        return self.nfl(pred, labels) + self.rce(pred, labels)

class NFLandMAE(torch.nn.Module):
    def __init__(self, alpha=1., beta=1., num_classes=10, gamma=0.5):
        super(NFLandMAE, self).__init__()
        self.num_classes = num_classes
        self.nfl = NormalizedFocalLoss(gamma=gamma, num_classes=num_classes, scale=alpha)
        self.mae = MAELoss(num_classes=num_classes, scale=beta)

    def forward(self, pred, labels):
        return self.nfl(pred, labels) + self.mae(pred, labels)


# Create your views here.
from django.shortcuts import render
from team.models import Player
from django.shortcuts import render, get_object_or_404, redirect, render_to_response
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger

def home(request):
    context = {'message': 'Here is a message!'}
    return render(request, "team/home.html", context)

def roster(request):
    player_list = Player.objects.all()
    paginator = Paginator(player_list, 100)
    page = request.GET.get('page')
    try:
        players=paginator.page(page)
    except PageNotAnInteger:
        players = paginator.page(1)
    except EmptyPage:
        players = paginator.page(1)
    return render(request, "team/roster.html", {'players': players})

def player(request, pk):
    player = get_object_or_404(Player, id=pk)
    return render(request, "team/player.html", {'player': player})


import numpy as np
import torch

from snownlp import SnowNLP
from common_utils import *
from preprocessing.clean_data import batchify

def get_overlap(list1, list2):
    """
    Returns a list of words that occur in both list1 and list2.
    Also returns total number of words in list1 and in list2 (can be used
    to compute similarity as a percentage, if desired)

    Args:
        list1 (string): filename corresponding to a list of words
        list2 (string): filename corresponding to a list of words

    Returns:
        overlap (list of string): words occuring in both list1 and list2
        len1 (int): number of words in list1
        len2 (int): number of words in list2
    """
    # File to List
    wordlist1 = file_to_str(list1)
    wordlist2 = file_to_str(list2)

    # Tokenize Lists
    s1 = SnowNLP(wordlist1)  # need to use SnowNLP for Chinese-character lists
    s2 = SnowNLP(wordlist2)

    # Get List Lengths
    wordlist1 = list(set(s1.words))
    wordlist2 = list(set(s2.words))
    len1 = len(wordlist1)
    len2 = len(wordlist2)

    # Count Overlapping Words
    # overlap = [w2 for w2 in wordlist2 if w2 in wordlist1]

    # Alternative to "Count Overlapping Words" with potentially better time complexity
    combined_wordset = set(wordlist1).union(set(wordlist2))
    overlap = len1 + len2 - len(combined_wordset)

    return overlap, len1, len2

def get_embedding_similarity(list1, list2, emb):
    """
    Given an embedding for a vocabulary and two lists of words from
    that vocabulary, return the cosine distance between the average
    word embeddings from each list.

    Args:
        list1 (string): filename corresponding to a list of words
        list2 (string): filename corresponding to a list of words
        emb (??): word embedding
    """
    raise NotImplementedError

def get_political_diff(list1, list2, model_file):
    # Load model and data
    model = torch.load(model_file)
    model.eval()
    data1 = batchify(list1)
    data2 = batchify(list2)

    # Inference
    positives1, positives2 = 0, 0
    for in1 in data1:
        output1 = model(in1[0])  # no labels
        positives1 += (output1.argmax(1) == 1).sum().item()
    for in2 in data2:
        output2 = model(in2[0])
        positives2 += (output2.argmax(1) == 1).sum().item() / len(data2)
    return positives1, positives2, len(data1.dataset), len(data2.dataset)

def get_political_ratio(list, model_file):
    # Load model and data
    model = torch.load(model_file)
    model.eval()
    data = batchify(list)

    # Inference
    positives = 0
    for in1 in data:
        output = model(in1[0])  # no labels
        positives += (output.argmax(1) == 1).sum().item()
    return positives / len(data.dataset)

def get_longest_subsequence_length(list1, list2):
    """
    Returns length of longest subsequence common to list1 and list2.
    Also returns total number of words in list1 and in list2 (can be used
    to compute similarity as a percentage, if desired)

    Args:
        list1 (string): filename corresponding to a list of words
        list2 (string): filename corresponding to a list of words

    Returns:
        length (int): length of longest subsequence
        len1 (int): number of words in list1
        len2 (int): number of words in list2
    """
    # File to List
    wordlist1 = file_to_str(list1)
    wordlist2 = file_to_str(list2)

    # Tokenize Lists
    s1 = SnowNLP(wordlist1)
    s2 = SnowNLP(wordlist2)

    # Get List Lengths
    wordlist1 = list(s1.words)
    wordlist2 = list(s2.words)
    len1 = len(wordlist1)
    len2 = len(wordlist2)

    length = longest_subsequence(wordlist1, wordlist2)

    return length, len1, len2

def longest_subsequence(list1, list2):
    """
    Return length of longest subsequence common to list1 and list2.

    Here, a subsequence is defined as a list of ordered entries
    occuring consecutively in a list.

    Example:
        [1,2,3] and [4] are subsequences of [1,2,3,4]
        [1,3,4] and [1,3,2] are not subsequences of [1,2,3,4]

    Args:
        list1 (list): a list whose elements can be of any class
            that implements __eq__
        list2 (list): a list whose elements are the same class as
            those of list1

    Returns:
        l (int): the length of the longest subsequence
    """
    # T[i, j] will store the length of longest substring
    # ending at list1[i] and list2[j]
    n = len(list1)
    m = len(list2)
    T = np.zeros((n, m))

    for i in range(n):
        for j in range(m):
            if list1[i] == list2[j]:
                if i == 0 or j == 0:
                    T[i, j] = 1
                else:
                    T[i, j] = T[i-1, j-1] + 1

    return np.max(T)

def _cosine_dist(v1, v2):
    return np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2))

def _euclidean_dist(v1, v2):
    return np.linalg.norm(v1 - v2)

def _manhattan_dist(v1, v2):
    raise np.linalg.norm(v1 - v2, ord=1)


import twitter

class pytwitter_forecast(NebriOS):
    listens_to = ['forecast_date']

    def check(self):
        return True

    def action(self):
        auth = twitter.OAuth(shared.ttoken, shared.ttoken_secret, shared.tconsumer_key, shared.tconsumer_secret)
        t = twitter.Twitter(auth=auth)
        status = "Forecast: " + self.check_city_forecast + " is " + self.forecast_text + " with temperature "+ self.forecast_lo + " - " + self.forecast_hi + " °C for " + self.forecast_date
        try:
            t.account.verify_credentials()
            try:
                t.statuses.update(status=status)
                # uncomment to update KVP of auth status for checking
                #self.pytwitter_update = "Run"
            except:
                self.pytwitter_update = "Fail"
        except:
            self.pytwitter_auth = "Fail"


import time
import threading
import logging
import traceback
import datetime
import os
import sys
import re
import robotparser as rp
import numpy as np
import random

import util
import decide
import queries
from conn import connect

class Crawler:
    '''
    Abstract class for crawling a news source.
    '''

    ########################
    #                      #
    #  Abstract functions  #
    #                      #
    ########################

    def sleep(self):
        '''
        Abstract function for waiting between requests.
        Can add additional functionality, such as random sleep times.
        '''
        return NotImplemented

    def is_article(self, url):
        '''
        Abstract function for determining if a url is an article or not.
        Returns True if the url is an article, false otherwise.
        '''
        return NotImplemented

    def extract_date_from_url(self, url):
        '''
        Abstract function for parsing a date from a url.
        Currently crawler only works for sources with dates in their urls.
        Takes a url as a string, returns a datetime.date object.
        '''
        return NotImplemented

    ############################
    #                          #
    #  Non-Abstract functions  #
    #                          #
    ############################

    def __init__(self, base_url_string):
        '''
        Not abstract.
        Parameters:
            base_url_string, a string representing the base url for a source (eg, foxnews.com)
            article_regex, a string representing a regex which only matches for articles
        '''
        self.base_url_string = base_url_string
        self.initialize_robots()

    def initialize_robots(self):
        '''
        Not abstract.
        Initializes a robot parser for the crawler.
        Use self.robot_parser.can_fetch("*", url) to decide if allowed or not.
        '''
        base_url_string = self.base_url_string
        robot_url = util.robots_url(base_url_string)
        robot_parser = rp.RobotFileParser()
        robot_parser.set_url(robot_url)
        robot_parser.read()
        self.robot_parser = robot_parser

    def decide_next_visit(self, conn, crawl_id, bad_urls):
        '''
        Not abstract.
        Decides which url to visit next.
        Returns a dictionary visit_url with two keys
            visit_url['id'] - database if of the url to visit
            visit_url['url'] - string representation of the url to visit
        Returns None if no urls left to visit.
        Strategy is to visit anything not visited this crawl, with the following priority:
            1) base url
            2) internal pages linked from the base url
            2) articles which haven't been visited yet, sorted by date
        Currently only implemented 1) and 2)
        '''
        base_url_string = self.base_url_string

        # strategy 1 - visit base url if not visited yet (ignore previous crawls)
        base_url_id = queries.insert_url(conn, base_url_string)
        base_url = {'id': base_url_id, 'url': base_url_string}
        visited_base = decide.visited_base_url(conn, crawl_id)
        if not visited_base:
            return base_url

        # strategy 2 - visit any urls linked by the base url that haven't been visited yet (ignore previous crawls)
        urls = decide.find_unvisited_links_from_base(conn, crawl_id, base_url_string)
        urls = filter(lambda url: self.robot_parser.can_fetch("*", url['url']), urls)
        urls = filter(lambda url: url['id'] not in bad_urls, urls)
        if len(urls) > 0:
            visit_url = random.choice(urls)
            return visit_url

        # strategy 3 - visit any articles not visited yet (including previous crawls), starting with the most recent
        urls = decide.find_unvisited_internal_urls(conn, base_url_string)
        urls = filter(lambda url: self.robot_parser.can_fetch("*", url['url']), urls)
        urls = filter(lambda url: url['id'] not in bad_urls, urls)
        urls = filter(lambda url: self.is_article(url['url']), urls)
        if len(urls) > 0:
            dates = map(lambda url: self.extract_date_from_url(url['url']), urls)
            reverse_sorted_dates = np.argsort(np.array(dates))[::-1]
            last_date_index = reverse_sorted_dates[0]
            visit_url = urls[last_date_index]
            return visit_url

        return None

    def crawl(self):
        '''
        Not abstract. Begins a crawl.
        Crawls until MAX_VISITS is reached, unless:
            - self.decide_next_visit(conn) returns None
            - Five exceptions in a row
        '''
        # initialize variables
        visits = 0
        MAX_VISITS = 1000 # so we don't just keep crawling forever
        bad_urls = set() # when a url doesn't work, add url_id to bad_urls, ignore in future
        error_count = 0
        base_url_string = self.base_url_string
        conn = connect()

        # initialize logging
        initialize_logging(base_url_string)
        start_time = time.time()
        logging.info('STARTING CRAWL AT TIME: {0}'.format(util.time_string(start_time)))

        # initlialize database for this crawl
        base_url_id = queries.insert_url(conn, base_url_string)
        source_id = queries.insert_source(conn, base_url_string)
        crawl_id = queries.insert_crawl(conn, base_url_string)

        while True:
            if error_count == 5:
                logging.error('Too many exceptions in a row, exiting.')
                break
            visit_url = self.decide_next_visit(conn, crawl_id, bad_urls)
            if visit_url is None:
                logging.info('Finished crawling, no more urls to visit.')
                break
            try:
                logging.info('Visiting {}'.format(visit_url['url']))
                self.visit(conn, crawl_id, source_id, visit_url)
                error_count = 0
            except Exception as e:
                logging.error('Error when downloading {0}'.format(visit_url['url']))
                logging.error(traceback.format_exc())
                bad_urls.add(visit_url['id'])
                error_count += 1
            visits += 1
            if visits == MAX_VISITS:
                logging.info('Finished crawling, reached max visits of {}'.format(MAX_VISITS))
                break
            self.sleep()

    def visit(self, conn, crawl_id, source_id, visit_url):
        '''
        Not abstract. Visits a url during a crawl.
        Inserts all relevant information to the database for a single visit.
        Inserts article information if the url matches the article regex.
        '''
        visit_url_id = visit_url['id']
        visit_url_string = visit_url['url']

        base_url_string = self.base_url_string
        html = util.download_html(visit_url_string)
        found_links = util.extract_links(html, base_url_string)
        visit_id = queries.insert_visit(conn, crawl_id, visit_url_id)
        new_url_ids = queries.insert_urls(conn, found_links)
        queries.insert_links(conn, visit_id, new_url_ids)

        if self.is_article(visit_url_string):
            article = util.extract_article(html, visit_url_string)
            article_title = article.title
            article_text = article.text
            article_date = self.extract_date_from_url(visit_url_string)
            queries.insert_article(conn, visit_url_id, article_title, article_text, article_date, source_id)

def initialize_logging(base_url):
    log_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'logs')
    if not os.path.exists(log_dir):
        os.makedirs(log_dir)
    source_str = util.extract_source(base_url)
    log_filename = 'LOG_{0}.log'.format(source_str)
    log_path = os.path.join(log_dir, log_filename)
    logging.basicConfig(filename=log_path, filemode='a', level=logging.INFO)
    stderrLogger=logging.StreamHandler()
    stderrLogger.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
    logging.getLogger().addHandler(stderrLogger)
    sys.excepthook = log_unchecked_exception

def log_unchecked_exception(exctype, value, tb):
    traceback.print_tb(tb)
    log_str = '''
UNCHECKED EXCEPTION
    Type: {}
    Value: {}
    Traceback: {}'''.format(exctype, value, traceback.print_tb(tb))
    logging.error(log_str)


import torch
import torch.nn as nn
import torch.nn.functional as F
from .utils import weight_reduce_loss

class FocalLoss(nn.Module):
    def __init__(self,
                 use_sigmoid=True,
                 gamma=2.0,
                 alpha=0.25,
                 reduction='mean',
                 loss_weight=1.0):
        super(FocalLoss, self).__init__()
        assert use_sigmoid is True, 'Only sigmoid focal loss supported now.'
        self.use_sigmoid = use_sigmoid
        self.gamma = gamma
        self.alpha = alpha
        self.reduction = reduction
        self.loss_weight = loss_weight

    def forward(self,
                pred,
                target,
                weight=None,
                avg_factor=None):
        device = pred.device
        assert pred.shape[0] == target.shape[0]

        if len(target.shape) == 1:
            target = torch.zeros(
                pred.shape,
                dtype=torch.long,
                device=device).scatter_(
                    1, target.view(-1, 1), 1)

        pred_sigmoid = pred.sigmoid()
        pred_sigmoid = torch.clamp(pred_sigmoid, 1e-4, 1.0 - 1e-4)
        target = target.type_as(pred)
        pt = (1 - pred_sigmoid) * target + pred_sigmoid * (1 - target)
        focal_weight = (self.alpha * target + (1 - self.alpha) *
                        (1 - target)) * pt.pow(self.gamma)
        loss = F.binary_cross_entropy_with_logits(
            pred, target, reduction='none') * focal_weight

        loss = torch.where(torch.ne(target, -1.0),
                           loss, torch.zeros(loss.shape).to(device))

        loss = self.loss_weight * weight_reduce_loss(
            loss,
            weight=weight,
            reduction=self.reduction,
            avg_factor=avg_factor)

        return loss


####################### IMPORT LIBRARIES ####################################
from pandas import ExcelFile, read_excel
from pandas import datetime
from sklearn.metrics import mean_squared_error
from math import sqrt
import matplotlib.pyplot as plt
import warnings
from hmmlearn.hmm import GaussianHMM
import numpy as np
import time

####################### FUNCTION TO READ FILE ################################
def readfile(coin_file, attribute):
    ''' Function to read and parse the data '''
    # Read excel file
    xls = ExcelFile(coin_file)
    # Date parser
    def parser(x):
        try:
            return datetime.strptime(str(x),'%Y-%m-%d %H:%M:%S')
        except:
            return datetime.strptime(str(x),'%Y-%m-%d')
    series = read_excel(xls, attribute, header = 0, parse_dates =[0], index_col = 0, squeeze = True, date_parser = parser)
    series = series.fillna(0)
    # Store in array
    X = series.values
    return X

###################### FUNCTION TO SPLIT THE DATA INTO TRAINING AND TEST SET #######
def train_test_split(X, fraction):
    ''' Function to split the data into training and test set '''
    # Train test split
    size = int(len(X)*fraction)
    train,test = X[0:size], X[size:len(X)]
    return train,test

##################### FUNCTION TO DEFINE GAUSSIAN HIDDEN MARKOV MODEL ###############
def GaussHMM(n_comp,cov_type,n_itr,train,num_samples_test):
    ''' Function to define Gaussian Hidden Markov Model '''
    # Reshape training data
    history = train.reshape(-1,1)
    # Gaussian hidden markov model
    hmm = GaussianHMM(n_components = n_comp, covariance_type = cov_type, n_iter = n_itr)
    with warnings.catch_warnings():
        warnings.simplefilter('ignore')
        hmm.fit(history)
    # Generate samples
    samples_test, _ = hmm.sample(num_samples_test)
    return samples_test

###################### FUNCTION TO CALCULATE PREDICTIONS AND EXPECTATIONS #############
def pred_expect(test,variable, samples_test, pred, expect):
    ''' Function to calculate predictions and expectations '''
    # Loop over test data
    for i in range(len(test)):
        # Integer values
        if variable == 'int':
            if round(samples_test[i][0]) < 0:
                pred.append(0)
            else:
                pred.append(round(samples_test[i][0]))
        # decimal values
        else:
            if samples_test[i][0] < 0:
                pred.append(0)
            else:
                pred.append(samples_test[i][0])
        expect.append(test[i])
    return pred, expect

############################# FUNCTION TO CALCULATE RMSE ###############################
def rmse(coin_file, attribute,iterations,fraction,n_comp,cov_type,n_itr,var):
    ''' Function to calculate RMSE '''
    X = readfile(coin_file,attribute)
    train,test = train_test_split(X,fraction)
    rmse_test = list()
    for j in range(iterations):
        ######################## FOR FINAL RMSE ############################
        pred = list()
        expect = list()
        num_samples_test = len(test) # number of samples to be generated
        samples_test = GaussHMM(n_comp,cov_type,n_itr,train,num_samples_test)
        pred,expect = pred_expect(test,var,samples_test, pred, expect)
        rmse_test.append(sqrt(mean_squared_error(pred,expect)))
    return rmse_test

##################### PRINT AVERAGE RMSE FOR TEST DATA #####################
coin_file = '4_RLC.xlsx'    # Filename
attribute = 'exchange'         # attribute
iterations = 30                 # number of iterations for averaging rmse
fraction = 0.80             # Train - test split
n_comp = 7                    # n_components for Gaussian HMM
cov_type = 'diag'                # covariance_type for Gaussian HMM
n_itr = 1000                  # n_iter for Gaussian HMM
var = 'float'                 # Integer or Float valued attribute

start = time.time()
rmse_test = rmse(coin_file, attribute,iterations,fraction,n_comp,cov_type,n_itr,var)
end = time.time()

print('The average RMSE over %d iterations is %.3f' %(iterations,np.array(rmse_test).mean()))
print('The time taken is %.3f seconds' %(end - start))


import time
from datetime import datetime, timedelta
from pydantic import BaseModel
from fastapi import FastAPI, Depends, File, UploadFile, HTTPException, Request, status
from fastapi.responses import HTMLResponse, JSONResponse, FileResponse
import uvicorn
import os
from pytube import YouTube
from pytube import Playlist
from pytube.cli import on_progress

app = FastAPI()

class Info(BaseModel):
    author: str
    title: str
    view: float
    length: int
    description: str
    thumbnail: str
    fileSizeHD: str

@app.get("/")
async def hello():
    return {"result": "Its working YES! This is a miracle!"}

@app.get("/info")
async def info(url: str = None):
    try:
        yt = YouTube(url)
        videoSize = yt.streams.get_highest_resolution()
        Info.author = yt.author
        Info.title = yt.title
        Info.view = yt.views
        Info.length = ("%.2f" % float(yt.length / 60))
        Info.description = yt.description
        Info.thumbnail = yt.thumbnail_url
        Info.fileSizeHD = str(round(videoSize.filesize * 0.000001, 2)) + " Mb"
        res = {None}
        for i in yt.streams:
            res.add(i.resolution)
        res.remove(None)
        res = [int(i) for i in [sub.replace('p', '') for sub in res]]
        sres = sorted(res)
        return { "Title": Info.title, "Resolution": sres, "Author": Info.author, "Thumbnail": Info.thumbnail, "View": Info.view,
                "Length": Info.length, "Description": Info.description,
                 "File size": Info.fileSizeHD}
    except Exception as e:
        return JSONResponse(
            status_code=status.HTTP_400_BAD_REQUEST,
            content={'message': str(e)})
    else:
        return JSONResponse(
            status_code=status.HTTP_200_OK,
            content={"result": 'success'})

@app.get("/video")
async def video(url: str = None):
    try:
        yt = YouTube(url, on_progress_callback=on_progress)
        yd = yt.streams.get_highest_resolution()
        folder_name = "FILE_NAME"
        file_path = os.getcwd() + "/" + folder_name
        video = yd.download(file_path)
        yt.title
        print(file_path)
        headers = {'success': f'video is ready, filename= {yt.title}'}
        return FileResponse(path=video, headers=headers, media_type='application/mp4', filename=(yt.title + ".mp4"))

    except Exception as e:
        return JSONResponse(
            status_code=status.HTTP_400_BAD_REQUEST,
            content={'message': str(e)}
        )
    else:
        return JSONResponse(
            status_code=status.HTTP_200_OK,
            content={"result": 'success, video is ready'}
        )

@app.get("/audio")
async def audio(url: str = None):
    try:
        ya = YouTube(url)
        folder_name = "FILE_NAME"
        # DEPENDS ON WHERE YOUR FILE LOCATES
        file_path = os.getcwd() + "/" + folder_name
        video = ya.streams.filter(only_audio=True).first()
        downloaded_file = video.download(file_path)
        base, ext = os.path.splitext(downloaded_file)
        audio = base + 'Audio.mp3'
        os.rename(downloaded_file, audio)
        ya.title
        print(file_path)
        print(audio)
        headers = {'success': f'audio is ready, filename= {ya.title}'}
        return FileResponse(path=audio, headers=headers, media_type='application/mp4', filename=(ya.title+".mp3"))
    except Exception as e:
        return JSONResponse(
            status_code=status.HTTP_400_BAD_REQUEST,
            content={'message': str(e)}
        )
    else:
        return JSONResponse(
            status_code=status.HTTP_200_OK,
            content={"result": 'success, audio is ready'}
        )

@app.get("/delete")
async def delete_files(min: int = 10):
    try:
        response_msg = []
        folder_name = "FILE_NAME"
        file_path = os.getcwd() + "/" + folder_name
        path = file_path
        files = os.listdir(file_path)
        print(files)
        print(f"file_path: {file_path}")
        file_name_delete = []
        dir_name = file_path
        # Get list of all files only in the given directory
        list_of_files = filter(lambda x: os.path.isfile(os.path.join(dir_name, x)),
                               os.listdir(dir_name))
        # Sort list of files based on last modification time in ascending order
        list_of_files = sorted(list_of_files,
                               key=lambda x: os.path.getmtime(os.path.join(dir_name, x))
                               )
        for file_name in list_of_files:
            file_path = os.path.join(dir_name, file_name)
            timestamp_str = time.strftime('%m/%d/%Y :: %H:%M:%S', time.gmtime(os.path.getmtime(file_path)))
            print(timestamp_str, ' -->', file_name)
            #filter by minite
            now = datetime.now()
            timestamp = datetime.timestamp(now)
            f = os.path.getmtime(file_path)
            file_date = datetime.fromtimestamp(f)
            file_date_age = file_date + timedelta(minutes=min)
            duration = now - file_date_age
            duration_in_s = duration.total_seconds()
            minut = (int(round(duration_in_s / 60, 0)))
            print(f"duration: {minut}")
            if minut > min:
                file_name_delete.append(file_name)
        ### create a list of old file by date
        print(f"file_name_delete {len(file_name_delete)}, {file_name_delete}")
        #print(os.listdir(path))
        for i in file_name_delete:
        #for file_name_delete in os.listdir(path):
            print(f"file_name: {file_name_delete}")
            #print(f"path: {path}")
            # construct full file path
            del_file = path + "/" + i
            #print(f"file: {del_file}")
            if os.path.isfile(del_file):
                response_msg.append(f"Deleting file: {del_file}")
                print('Deleting file:', del_file)
                os.remove(del_file)
                print(len(del_file))
        if len(response_msg) < 1:
        #     response_msg.clear()
            response_msg.append(f"no files to delete after: {min} min")
        return response_msg
    except Exception as e:
        return JSONResponse(
            status_code=status.HTTP_400_BAD_REQUEST,
            content={'message': str(e)})
    else:
        return JSONResponse(
            status_code=status.HTTP_200_OK,
            content={"result": 'success'})

@app.get("/aws")
async def aws_files():
    folder_name = "FILE_NAME"
    file_path = os.getcwd() + "/" + folder_name
    files = os.listdir(file_path)
    result = []
    dir_name = file_path
    # Get list of all files only in the given directory
    list_of_files = filter(lambda x: os.path.isfile(os.path.join(dir_name, x)),
                           os.listdir(dir_name))
    # Sort list of files based on last modification time in ascending order
    list_of_files = sorted(list_of_files,
                           key=lambda x: os.path.getmtime(os.path.join(dir_name, x))
                           )
    for file_name in list_of_files:
        file_path = os.path.join(dir_name, file_name)
        size = ("%.2f" % float(os.path.getsize(file_path)*0.000001))
        timestamp_str = time.strftime('%m/%d/%Y :: %H:%M:%S', time.gmtime(os.path.getmtime(file_path)))
        result.append(f"({timestamp_str}, ' -->', {file_name}, ' -->', {size}' MB')")
    return result

def raise_exception():
    return HTTPException(status_code=404,
                         detail="Input is Not valid!",
                         headers={"X-Header_Error":
                                  f"Nothing to be seen"})

if __name__ == "__main__":
    uvicorn.run("main:app", host="127.0.0.1", port=5000, reload=True, log_level="info", workers=2)

print('Download Complete')


import argparse
import gym
import random
import tensorflow as tf
import numpy as np
from tqdm import trange
from tensorflow import keras
from network import SharedModel
from subproc_env import EnvActor, SubProcessEnv

# Some parameters taken from OpenAI
# baselines implementation, since
# they're not mentioned in the paper.
num_actors = 8
# Values taken from Atari experiments
# in original paper where relevant
gae_lambda = 0.95
gamma = 0.99
base_clip_epsilon = 0.1
max_steps = 1e6
base_learning_rate = 2.5e-4
horizon = 128
batch_size = 32
optim_epochs = 3
value_loss_coefficient = 1
entropy_loss_coefficient = .01
gradient_max = 10.0
start_t = 0
checkpoint_filename = "./ppo-model.ckpt"
log_dir = "./tb_log"
SMALL_NUM = 1e-8

def main():
    env_name = "PongNoFrameskip-v4"
    # NOTE: This is currently not used since we use SubProcessEnv instead;
    # only used for getting shape of observation/acton space.
    unused_env = gym.make(env_name)
    #pobs_shape = unused_env.observation_space.shape
    # Hard-coding pre-processing step shape; could read it from an example output instead?
    obs_shape = (84, 84, 4)
    num_actions = unused_env.action_space.n
    model = SharedModel(obs_shape, num_actions)

    t = start_t
    last_save = 0
    actors = []

    for ii in range(num_actors):
        actors.append(EnvActor(SubProcessEnv(env_name), model, num_actions))

    while(t <= max_steps):
        for ii in range(horizon):
            for actor in actors:
                actor.step_env(t)
            t += 1

        for actor in actors:
            actor.calculate_horizon_advantages(t)

        # Construct randomly sampled (without replacement) mini-batches.
        obs_horizon = []
        act_horizon = []
        policy_horizon = []
        adv_est_horizon = []
        val_est_horizon = []

        for actor in actors:
            obs_a, act_a, policy_a, adv_est_a, val_est_a = actor.get_horizon(t)
            obs_horizon.extend(obs_a)
            act_horizon.extend(act_a)
            policy_horizon.extend(policy_a)
            adv_est_horizon.extend(adv_est_a)
            val_est_horizon.extend(val_est_a)

        # Normalizing advantage estimates.
        # NOTE: Adding this significantly improved performance
        # NOTE: Moved this out of each individual actor, so that advantages for the whole batch are normalized with each other.
        adv_est_horizon = np.array(adv_est_horizon)
        adv_est_horizon = (adv_est_horizon - np.mean(adv_est_horizon)) / (np.std(adv_est_horizon) + SMALL_NUM)

        num_samples = len(obs_horizon)
        indices = list(range(num_samples))
        for e in range(optim_epochs):
            random.shuffle(indices)
            ii = 0
            # TODO: Don't crash if batch_size is not a divisor of total sample count.
            while ii < num_samples:
                obs_batch = []
                act_batch = []
                policy_batch = []
                adv_batch = []
                value_sample_batch = []

                for _ in range(batch_size):
                    index = indices[ii]
                    obs_batch.append(obs_horizon[index].__array__())
                    act_batch.append(act_horizon[index].__array__())
                    policy_batch.append(policy_horizon[index].__array__())
                    adv_batch.append(adv_est_horizon[index].__array__())
                    value_sample_batch.append(val_est_horizon[index])
                    ii += 1

                def alpha_anneal(t):
                    return np.maximum(1.0 - (float(t) / float(max_steps)), 0.0)

                total_loss = model.train(np.array(obs_batch),
                            np.array(act_batch),
                            np.array(policy_batch),
                            np.array(adv_batch),
                            np.array(value_sample_batch),
                            alpha_anneal(t))

        for actor in actors:
            actor.flush(t)

        if t-last_save > 10000:
            print("Saving network")
            model.network.save(checkpoint_filename)
            last_save = t

        all_ep_rewards = []
        for actor in actors:
            all_ep_rewards.extend(actor.episode_rewards)

        if len(all_ep_rewards) >= 10:
            print("T: %d" % (t,))
            print("\tAVG Reward: %f" % (np.mean(all_ep_rewards),))
            print("\tMIN Reward: %f" % (np.amin(all_ep_rewards),))
            print("\tMAX Reward: %f" % (np.amax(all_ep_rewards),))
            for actor in actors:
                actor.episode_rewards = []

if __name__ == '__main__':
    main()


from flask import Flask,request
import sqlite3

app = Flask(__name__)

connection = sqlite3.connect('sms.db')
curser = connection.cursor()
curser.execute('create table if not exists students (sid integer primary key,name text,age integer,address text)')
connection.close()

@app.route('/student_details')
def details():
    connection = sqlite3.connect('sms.db')
    curser = connection.cursor()
    return {'students' : list(curser.execute('select * from students'))}

@app.route('/details',methods = ['POST'])
def register():
    data = request.get_json()
    connection = sqlite3.connect('sms.db')
    curser = connection.cursor()
    curser.execute("insert into students values('{},{},{},{}')".format(data['sid'],data['name'],data['age'],data['address']))
    connection.commit()
    connection.close()
    return 'User Created Successfully.. '
app.run(port=5000)


# 数据预处理 阴影过滤
#yangzhen
#2020.4.13
#translate from matlab
"""get the shadow proportion from images
   of remote sensing"""
import numpy as np
import cv2
import os
import json
from shutil import copyfile
import argparse

def cv_imread(file_path):
    cv_img=cv2.imdecode(np.fromfile(file_path,dtype=np.uint8),1)
    return cv_img

def cv_imwrite(filepath,img):
    cv2.imencode(".png",img)[1].tofile(filepath)

def standard(data):
    '''影像文件标准化
       输入单通道影像
       输出标准化后单通道影像'''
    mdata = data.copy()
    irow, icol = mdata.shape[0:2]
    mdata = np.reshape(mdata, [irow*icol, 1])
    temp1 = mdata - np.min(data)
    result = temp1/(np.max(data)-np.min(data))
    result = np.reshape(result, [irow, icol])
    np.seterr(divide='ignore', invalid='ignore')
    return result

def GetLight(img):
    '''计算人眼视觉特性亮度'''
    mimg = img.copy()
    B = mimg[:,:,0]
    G = mimg[:,:,1]
    R = mimg[:,:,2]
    result = 0.04*R+0.5*G+0.46*B
    return result

def GetColor(img):
    '''色度空间归一化'''
    mimg = img.copy()
    misc = mimg[:,:,0]+mimg[:,:,1]+mimg[:,:,2]
    misc[misc == 0] = 0.0000001
    mimg[:,:,0] = img[:,:,0]/misc
    mimg[:,:,1] = img[:,:,1]/misc
    result = np.abs(mimg - img)
    result = (result[:,:,0]+result[:,:,1])/2
    return result

def GetVege(img):
    '''获取植被特征'''
    mimg = img.copy()
    B = mimg[:,:,0]
    G = mimg[:,:,1]
    R = mimg[:,:,2]
    result = G-np.minimum(R, B)
    result[result<0] = 0
    return result

def GetLDV(idist, ilight, ivege):
    '''总决策'''
    idist = standard(idist)
    ilight = standard(ilight)
    ivege = standard(ivege)
    result = idist-ilight-ivege
    result[result<0]=0
    return result

def FinalTrare(img):
    '''结果后处理'''
    mimg = img.copy()
    mimg = np.uint8(standard(mimg)*255)
    T, result = cv2.threshold(mimg, 0, 255, cv2.THRESH_OTSU)
    result = cv2.medianBlur(result, 7)
    return result

def ShadowsProportion(path:{}):
    """
       阴影提取
       @@path: {},
       @path[0] 待检测阴影的影像
       @path[1] 待检测阴影的影像
       @path[2]  阴影比例阈值
       ps: 当两张影像过大时会进行分块
    """
    File_in = path[0]
    File_out = path[1]
    T = path[2]
    mpath = path[3]
    File_out2=path[4]
    if not os.path.exists(File_out):
        os.makedirs(File_out)
    if not os.path.exists(File_out2):
        os.makedirs(File_out2)
    #开始检测
    namelist=[]
    for filename in os.listdir(File_in):
        if not filename.find('.png') == -1:
            namelist.append(filename)
    n = len(namelist)
    fid = open('ShadowsProportion.txt', 'w')
    for i in range(n):
        filenamein = os.path.join(File_in, namelist[i])
        img = cv_imread(filenamein)
        #获取阴影
        img1 = img.astype(np.float)
        img1[:,:,0] = standard(img[:,:,0])
        img1[:,:,1] = standard(img[:,:,1])
        img1[:,:,2] = standard(img[:,:,2])
        idist = GetColor(img1)
        ilight = GetLight(img1)
        ivege = GetVege(img1)
        final = GetLDV(idist, ilight, ivege)
        shadow = FinalTrare(final)
        shadow = shadow/255
        #计算阴影比例并保存比例值
        S = shadow.size
        s = np.sum(sum(shadow))
        iratio = s/S
        fid.write(namelist[i] + ',' + str('%.3f' % iratio) + '\n')
        #保存阴影比例小于阈值的图片
        filenameout = os.path.join(File_out, namelist[i])
        filenameout2 = os.path.join(File_out2, namelist[i])
        mapout = mpath.replace('rawdata','noshade')
        if not os.path.exists(mapout):
            os.makedirs(mapout)
        if iratio < T:
            cv_imwrite(filenameout, img)
            copyfile(os.path.join(mpath,namelist[i]),(os.path.join(mapout,namelist[i])))
        else :
            cv_imwrite(filenameout2, img)
    fid.close()

def takejson(getjson):
    json1 = json.loads(getjson)

    path = {}
    path[0] = json1['rpath']
    path[1] = path[0].replace('rawdata','noshade')
    path[2] = json1['shadowProportion']
    path[3]=json1['mpath']
    path[4]=path[0].replace('rawdata','withshade')
    #print (path)
    ShadowsProportion(path)

    print ('Shadow filtering Completed')

if __name__ == "__main__":
    #获取输入图片路径，阴影比例阈值，输出图片路径
    # File_in = input('Please input the data file name:')
    # T = float(input('Please input the  threshold value:'))
    # File_out = input('Please input the out-img filename:')

    parser = argparse.ArgumentParser()
    parser.add_argument('--input_json', type=str, help='输入json字符串')
    args = parser.parse_args()

    # json1={'rpath':r'F:\Chicago2\metadata\谷歌影像无标注\14\14_14aligned','mpath':r'F:\Chicago2\metadata\谷歌地图无标注\14\14_14aligned','shadowProportion':0.2}
    # getjson=json.dumps(json1)
    takejson(args.input_json)


#!/usr/bin/env python3

import argparse
import random
import json
import re
import importlib.util
import os.path
import sys
import types
import inspect
import pandas as pd
import numpy as np

MAX_SLOT=8

SMART_COMMENT="\\s*#+\\s*(fastscore|odg)\\.(\\S*)\\s*:\\s*(\\S*)\\s*$"

def is_input_slot(s):
	return s % 2 == 0

## Parse command line
parser = argparse.ArgumentParser()
parser.add_argument("source_file", metavar="MODEL")
parser.add_argument("-i", "--input:0", action="store", help="Name of the input file (slot 0)", metavar="FILE")
parser.add_argument("-o", "--output:1", action="store", help="Name of the output file (slot 1)", metavar="FILE")
parser.add_argument("-b", "--batch-size", action="store", type=int, default=10, help="Name of the output file (slot 1)", metavar="NUM")
#parser.add_argument("-v", "--verbose", action="store_true", help="Increase verbosity")
# Add more --input/--output options
for s in range(2,MAX_SLOT):
	tag = "input" if is_input_slot(s) else "output"
	parser.add_argument("--{}:{}".format(tag,s), action="store", help="Name of the {} file (slot {})".format(tag,s), metavar="FILE")

args = parser.parse_args()

def check_scope(scope):
	if scope in ["$all", "$in", "$out"]:
		return scope
	else:
		try:
			return int(scope)
		except ValueError:
			print("Smart comment not recognized")

def default_scope(item):
	if item == "recordsets":
		return "$all"
	elif item == "action":
		return "$in"
	elif item == "slot":
		return "$all"
	elif item == "schema":
		return "$all"
	else:
		return None

def parse_comments1(line, slots):
	if re.match(SMART_COMMENT, line):
		tokens = re.split(SMART_COMMENT,line)
		item0=tokens[2]
		value=tokens[3]
		x=item0.split(".")
		item=x[0]
		scope = check_scope(x[1]) if len(x) > 1 else default_scope(item)
		for s in range(0,MAX_SLOT):
			if scope is None:
				continue
			is_deprecated=False
			if (scope == "$all" or
					scope == "$in" and is_input_slot(s) or
					scope == "$out" and not is_input_slot(s) or
					scope == s):
				if (item == "recordsets"):
					if (value in ["both","none","input","output"]):
						# deprecated style
						if scope != "$all":
							sys.exit("Invalid scope")
						is_deprecated=True
						if (s == 0 or s == 1):
							if (value == "both"):
								slots[s]['recordsets'] = True
							elif (value == "none"):
								slots[s]['recordsets'] = False
							elif (value == "input"):
								slots[s]['recordsets'] = (s == 0)
							elif (value == "output"):
								slots[s]['recordsets'] = (s == 1)
					else:
						# new style
						flag = None
						if (value == "true" or value == "yes"):
							flag = True
						elif (value == "false" or value == "no"):
							flag = False
						else:
							sys.exit("Value '{}' not recognized (use 'true', 'false', 'yes', or 'no')".format(value))
						slots[s]['recordsets']=flag
				elif (item == "action"):
					if not is_input_slot(s):
						sys.exit("An action callback being assigned to an output slot {}".format(s))
					slots[s]['action'] = None if (value == "unused") else value

				# Any mention of a slot makes it active
				if (value != "unused"):
					if (not is_deprecated or s == 0 or s == 1):
						slots[s]['active'] = True

				if (item == "slot"):
					if (value != "unused"):
						sys.exit("Value '{}' not supported (set to 'unused' to disable the slot)".format(value))
					slots[s]['active'] = False

def parse_comments():
	slots = []
	for s in range(0,MAX_SLOT):
		slots.append({
			'action': None,
			'recordsets': False,
			'active': False,
			'file': None
		})
		if is_input_slot(s):
			slots[s]['action'] = "action"

	# By default, slots 0 and 1 are active
	slots[0]['active'] = True
	slots[1]['active'] = True

	f = open(args.source_file)
	for l in f:
		parse_comments1(l, slots)
	f.close()

	return slots

model_slots = parse_comments()

# Collect all data-related options
for k in args.__dict__:
	if (k.startswith("input:") or k.startswith("output:")):
		s = int(k.split(":")[1])
		data_file = args.__dict__[k]
		if not data_file is None:
			if (is_input_slot(s) and not os.path.isfile(data_file)):
				sys.exit("{} not found".format(data_file))
			model_slots[s]['file'] = data_file

# Either all or none input slots must have action set
all_actions=True
none_actions=True
for s in range(0,MAX_SLOT):
	if (not is_input_slot(s)):
		continue
	if (model_slots[s]['action'] is None):
		all_actions=False
	else:
		none_actions=False

if (not all_actions and not none_actions):
	sys.exit("Either all input slots must have action callbacks set or none of them should")

# Check for dangling slots/files
for s in range(0,MAX_SLOT):
	active=model_slots[s]['active']
	data_file=model_slots[s]['file']
	if (active and data_file is None and s != 0 and s != 1):
		sys.exit("Model uses slot {} but there is no data file attached to it".format(s))
	if (not active and not data_file is None):
		sys.exit("Model does not use slot {} but the data file {} is attached to it".format(s,data_file))

inputs=[]
outputs=[]
for s in range(0,MAX_SLOT):
	if not model_slots[s]['active']:
		continue
	if is_input_slot(s):
		inputs.append({
			'slot': s,
			'seq_no': 1,
			'conn': None,
			'entry': None
		})
	else:
		outputs.append({
			'slot': s,
			'conn': None
		})

# Open input files
for i in inputs:
	s = i['slot']
	data_file = model_slots[s]['file']
	conn = open(0) if (s == 0 and data_file is None) else open(data_file)
	i['conn'] = conn

# Open output files
for i in outputs:
	s = i['slot']
	data_file = model_slots[s]['file']
	conn = open(1,'w') if (s == 1 and data_file is None) else open(data_file,'w')
	i['conn'] = conn

# Read a batch of records from the slot
def read_records(s):
	ii = None
	for i in range(0,len(inputs)):
		if inputs[i]['slot'] == s:
			ii = i
			break
	if ii is None:
		if model_slots[s]['active']:
			return None
		else:
			sys.exit("Slot {} is not in use".format(s))

	conn = inputs[ii]['conn']
	seq_no = inputs[ii]['seq_no']

	records = []
	at_eof = False
	while len(records) < args.batch_size:
		l = conn.readline()
		if len(l) == 0:
			if conn.name != 0:
				conn.close()
			inputs.pop(ii)
			at_eof = True
			break

		x = json.loads(l)
		#TODO
    #if (is.list(x) && x$`$fastscore` == "set"):
    #	break
		records.append(x)

	if at_eof and len(records) == 0:
		return []

	if not at_eof:
		old_seq_no = seq_no
		seq_no = seq_no + len(records)
		inputs[ii]['seq_no'] = seq_no

	return records

def as_recordset(records):
	if records is None:
		return None

	return records

	#TODO
	#if (len(records) == 0):
	#	pd.DataFrame(records)
	#else:
	#	head = records[0]

class Slot(object):
	def __init__(self, n):
		self.n = n

	def __iter__(self):
		return self

	def __next__(self):
		data = self.read()
		if data is None:
			raise StopIteration
		else:
			return data

	def read(self):
		if (not is_input_slot(self.n)):
			sys.exit("Model attempts to explicitly read from an output slot {}".format(self.n))
		records = read_records(self.n)
		data = as_recordset(records)
		return data

	def write(self, rec):
		if (is_input_slot(self.n)):
			sys.exit("Model emits data to an input slot {}:".format(s))
		conn = None
		for i in outputs:
			if i['slot'] == self.n:
				conn = i['conn']
				break
		if conn is None:
			sys.exit("Model emits data to an unknown slot {}".format(self.n))

		recordsets = model_slots[self.n]['recordsets']

		if not recordsets:
			print(json.dumps(rec), file=conn)
		else:
			sys.exit("TOODOO")

fio = types.ModuleType("fastscore.io")
fio.__dict__['Slot'] = Slot
sys.modules["fastscore.io"] = fio

spec = importlib.util.spec_from_file_location('model', args.source_file)
mod = importlib.util.module_from_spec(spec)
# Callbacks not resolved yet
spec.loader.exec_module(mod)

# TODO Check/wrap action callbacks
for s in range(0,MAX_SLOT):
	slot = model_slots[s]
	if not slot['active']:
		continue
	if not slot['action'] is None:
		entry = getattr(mod,slot['action'])
		if entry is None:
			sys.exit("A slot {} callback function named '{}' not found".format(s,slot['action']))
		if not isinstance(entry, types.FunctionType):
			sys.exit("A slot {} callback named '{}' must be a function".format(s,slot['action']))
		sig = inspect.signature(entry)
		arity = len(sig.parameters)
		if (arity < 1 or arity > 3):
			sys.exit("A slot {} callback function named '{}' must have arity 1, 2, or 3 (not {})".format(s,slot['action'],arity))
		wrapped_entry = entry
		if arity == 1:
			wrapped_entry = lambda data,slot,seqno: entry(data)
		elif arity == 2:
			wrapped_entry = lambda data,slot,seqno: entry(data,seqno)

		for i in inputs:
			if i['slot'] == s:
				i['entry'] = wrapped_entry

model_uses_callbacks = False
if len(inputs) > 0:
	model_uses_callbacks = isinstance(inputs[0]['entry'], types.FunctionType)

if model_uses_callbacks:
	while len(inputs) > 0:
		# Pick a random slot
		select = random.choice(inputs)
		s = select['slot']

		seq_no = select['seq_no']
		records = read_records(s)

		action = model_slots[s]['action']
		recordsets = model_slots[s]['recordsets']

		if recordsets:
			sys.exit("TODO recordests")
		else:
			# Invoke the callback for each record
			for rec in records:
				print(*select['entry'](rec, s, seq_no)) ## calls 'action'
				seq_no = seq_no + 1

Product Manuals/Model Launchers/Python Launcher/lh.py


"""
N×M 크기의 공간에 아기 상어 여러 마리가 있다. 공간은 1×1 크기의 정사각형 칸으로 나누어져 있다. 한 칸에는 아기 상어가 최대 1마리 존재한다.

어떤 칸의 안전 거리는 그 칸과 가장 거리가 가까운 아기 상어와의 거리이다. 두 칸의 거리는 하나의 칸에서 다른 칸으로 가기 위해서 지나야 하는 칸의 수이고, 이동은 인접한 8방향(대각선 포함)이 가능하다.

안전 거리가 가장 큰 칸을 구해보자.
"""

import sys
from collections import deque
N, M = list(map(int, sys.stdin.readline().strip().split()))
blocks = [[0] * M for _ in range(N)]  # blocks
dir = [(0, -1), (-1, -1), (-1, 0), (-1, 1), (0, 1), (1, 1), (1, 0), (1, -1)]

for i in range(N):
    blocks[i] = list(map(int, sys.stdin.readline().strip().split()))

def out_of_range(y,x):
    return y < 0 or x < 0 or y >= N or x >= M

def solve():
    max_val = float("-inf")
    for i in range(N):
        for j in range(M):
            if blocks[i][j] == 0:
                # bfs 시작
                q = deque()
                visited = set()
                q.append((i,j, 0))
                visited.add((i,j))
                dist = 0
                while q:
                    row, col, distance = q.popleft()
                    # print(row, col, distance)
                    if blocks[row][col] == 1:
                        dist = distance
                        break
                    for idx in range(8):
                        next_r, next_c = row + dir[idx][0], col + dir[idx][1]
                        if out_of_range(next_r, next_c) or (next_r, next_c) in visited:# or (next_r, next_c) in did_shark_test:
                            continue
                        q.append((next_r, next_c, distance + 1))
                        visited.add((next_r,next_c))
                max_val = max(max_val, dist)
    return max_val

def main():
    print(solve())

main()