from glob import glob

import motmetrics as mm
import cv2

from utils import get_gt_from_file, Sampler, trans, load_descriptors, assign_ids
from nms import nms
threshold = 0.90

def get_metrics(hypothesis_bboxes, hypothesis_ids, gt_path, acc):
    """
    :param hypothesis_bboxes:
    :param hypothesis_ids:
    :param gt_path: path to the gt file
    :return: nothingness
    """
    gt_bboxes, gt_ids = get_gt_from_file(gt_path)

    for gt_bboxes_frame, gt_ids_frame, hyp_bboxes_frame, hyp_ids_frame in zip(gt_bboxes, gt_ids,
                                                                              hypothesis_bboxes, hypothesis_ids):
        distances_frame = mm.distances.iou_matrix(gt_bboxes_frame, hyp_bboxes_frame)
        acc.update(gt_ids_frame, hyp_ids_frame, distances_frame)

num_descriptors = 361
epoch_number = 60
is_test = False

if is_test:
    gt_path = 'datasets/rrc-text-videos/ch3_test/'  # test
    descriptors_path = 'extracted_descriptors/extracted_descriptors_'+ str(num_descriptors) +'_test'  # test
else:
    gt_path = 'datasets/rrc-text-videos/ch3_train/'  # train
    descriptors_path = 'extracted_descriptors/extracted_descriptors_'+ str(num_descriptors) + '_dist' # train

annotations_paths = glob(gt_path + '*.xml')
# annotations_path = ["datasets/rrc-text-videos/ch3_test/Video_49_6_4_GT.xml"]

sampler = Sampler(weights_path='models/models_361_dropout/model-epoch-' + str(epoch_number) + '.pth', num_descriptors=num_descriptors,
                  hidden_size=256, input_size=6)

with open("results-" + str(num_descriptors) + "-" + str(epoch_number) + ".txt", "a") as f:
    if is_test:
        f.write("Results for test split\n")
    else:
        f.write("Results for train split\n")

for annotations_path in annotations_paths:
    acc = mm.MOTAccumulator(auto_id=True)

    with open("results-" + str(num_descriptors) + "-" + str(epoch_number) + ".txt", "a") as f:
        f.write("Results for file " + annotations_path + "\n")

    video_path = annotations_path.replace("_GT.xml", ".mp4")
    video_name = video_path.split('/')[-1].replace('.mp4', '')
    if is_test:
        voc_path = annotations_path.replace("GT.xml", "GT_voc.txt")  # test
    else:
        voc_path = annotations_path.replace("GT.xml", "GT_voc.txt")  # test
        # voc_path = annotations_path.replace("GT.xml", "GT.txt")  # train

    cap = cv2.VideoCapture(video_path)
    width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
    height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
    num_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))

    predicted_bboxes = [[] for dim in range(num_frames)]
    predicted_ids = [[] for dim in range(num_frames)]

    with open(voc_path) as f:
        lines = f.readlines()

    for line in lines:
        line = line.split(',')[-1]
        word = line.translate(trans).lower()
        predictions = sampler.sample_video(word, video_name, descriptors_path=descriptors_path)
        predictions = predictions.cpu().squeeze(0)

        predicted_bboxes_word = [[] for dim in range(num_frames)]

        descriptors = load_descriptors(video_name, word, descriptors_path, num_descriptors=num_descriptors)
        for frame in range(num_frames):
            predicted_bboxes_frame = []
            predicted_scores = []
            for j in range(descriptors.shape[1]):
                if predictions[frame, j] > 0.5:
                    center_x, center_y, w, h, p, _ = descriptors[frame, j]
                    center_x *= width
                    w *= width
                    center_y *= height
                    h *= height
                    x = center_x - w / 2.
                    y = center_y - h / 2.
                    predicted_scores.append(predictions[frame, j])
                    predicted_bboxes_frame.append([x, y, x+w, y+h])

            indices = nms.boxes(predicted_bboxes_frame, predicted_scores)  # non maximal suppresion
            for index in indices:
                predicted_bboxes_word[frame].append(predicted_bboxes_frame[index])

        tracked_predictions_word = assign_ids(predicted_bboxes_word)
        for i, tracked_predictions_frame in enumerate(tracked_predictions_word):
            for tracked_prediction in tracked_predictions_frame:
                predicted_bboxes[i].append(list(tracked_prediction[0:4]))
                predicted_ids[i].append(int(tracked_prediction[4]))

    get_metrics(predicted_bboxes, predicted_ids, annotations_path, acc)
    mh = mm.metrics.create()

    try:
        summary = mh.compute_many(
            [acc],
            metrics=mm.metrics.motchallenge_metrics,
            names=['full'])

        strsummary = mm.io.render_summary(
            summary,
            formatters=mh.formatters,
            namemap=mm.io.motchallenge_metric_names)

        with open("results-" + str(num_descriptors) + "-" + str(epoch_number) + ".txt", "a") as f:
            f.write(strsummary + '\n')
    except:
        with open("results-" + str(num_descriptors) + "-" + str(epoch_number) + ".txt", "a") as f:
            f.write('woops\n')


import sys
from collections import deque

def BFS(baked, starts, n, m):
    flag = False  # 익을 수 있는 모든 토마토가 익은 후, 0이 남아있다면 True
    max_cnt = 0  # 가장 오래 걸리는 일수 저장
    q = deque(starts)
    dr = [(-1, 0), (1, 0), (0, -1), (0, 1)]

    while q:
        i, j = q.popleft()
        for di, dj in dr:
            if 0 <= i+di < n and 0 <= j+dj < m and not baked[i+di][j+dj]:
                baked[i+di][j+dj] = baked[i][j] + 1
                q.append((i+di, j+dj))
                max_cnt = max(max_cnt, baked[i+di][j+dj])

    for i in range(N):
        for j in range(M):
            if not baked[i][j]:
                flag = True
                break

    return -1 if flag else (max_cnt-1 if max_cnt > 0 else 0)

M, N = map(int, sys.stdin.readline().split())
tomato_info = [list(map(int, sys.stdin.readline().split())) for _ in range(N)]
start_positions = []
for i in range(N):
    for j in range(M):
        if tomato_info[i][j] == 1: start_positions.append((i, j))

print(BFS(tomato_info, start_positions, N, M))


import json
import numpy as np

                                                        ### Tree structs and functions ###
class Node:

  # constructor arguments
  # msg -> json object  ex.{"TEMP": 25,"HUM": 50}
  def __init__(self, msg):
    self.msg = msg
    self.left = None
    self.right = None

# desc  -> insert new node to root
# root -> root of the current tree
# msg  -> json object  ex.{"TEMP": 25,"HUM": 50}
# void functions no return
def insert_left(root, msg):
  root.left = Node(msg)

def insert_right(root, msg):
  root.right = Node(msg)

# desc  -> insert a whole subtree to root
# root      -> pointer to root of the current tree
# subParent -> pointer to root of the sub tree to be appended to root
# reutrns   -> nothing
def insert_parent_right(root, subParent):
  root.right = subParent

def insert_parent_left(root, subParent):
  root.left = subParent

                                                         ### end tree ###

                                                    ### utility functions ###

# desc  -> calculate the lev distance of two strings
# token1   -> string of type TAG1-TAG2-...-TAGN
# token2   -> string of type TAG1-TAG2-...-TAGN
# returns  -> float (numeric value of difference between the 2 tokens)
def levenshtein_distance(token1, token2):
  distances = np.zeros((len(token1) + 1, len(token2) + 1))

  for t1 in range(len(token1) + 1):
    distances[t1][0] = t1

  for t2 in range(len(token2) + 1):
    distances[0][t2] = t2

  a = 0
  b = 0
  c = 0

  for t1 in range(1, len(token1) + 1):
    for t2 in range(1, len(token2) + 1):
      if (token1[t1 - 1] == token2[t2 - 1]):
        distances[t1][t2] = distances[t1 - 1][t2 - 1]
      else:
        a = distances[t1][t2 - 1]
        b = distances[t1 - 1][t2]
        c = distances[t1 - 1][t2 - 1]

        if (a <= b and a <= c):
          distances[t1][t2] = a + 1
        elif (b <= a and b <= c):
          distances[t1][t2] = b + 1
        else:
          distances[t1][t2] = c + 1

  return distances[len(token1)][len(token2)]

# desc  -> print the tree using lever order traversal
# root     -> pointer to root of the tree
# returns  -> nothing (display function)
def level_order_traversal(root):
  if not root:
    return

  queue = [root]

  while queue:
    level_size = len(queue)
    level_vals = []

    for i in range(level_size):
      node = queue.pop(0)
      level_vals.append(node.msg)
      if node.left:
        queue.append(node.left)
      if node.right:
        queue.append(node.right)

    print(" ".join(map(str, level_vals)))

# args for distance_matrix_calculation
# tag_arr     -> array of strings
# table_size  -> integer value of the size of the tag_arr
# returns     -> 2d array of distances
def distance_matrix_calculation(tag_arr, table_size):
  distances = [[0 for j in range(table_size)] for i in range(table_size)]

  #populate the upper triangular part of the matrix with the distances between the tags
  for i in range(table_size):
    for j in range(i + 1, table_size):
      distances[i][j] = levenshtein_distance(tag_arr[i], tag_arr[j])

  return distances

# desc  -> merges two json obects keeping the average of their values in case of two same instances
# obj1  -> json object  ex.{"TEMP": 25,"HUM": 50}
# obj2  -> json object  ex.{"TEMP": 25,"HUM": 50}
# returns     -> 2d array of distances
def message_merge(obj1, obj2):
  result = obj1.copy()

  for key, value in obj2.items():
      if key in result:
          result[key] = (result[key] + value) / 2
      else:
          result[key] = value

  return result

# desc  -> finds and returns and object in the array
# json_arr          -> array of json objects
# string_to_search  -> string of tags seperated by '-'
# returns           -> json object
def message_return(json_arr, string_to_search):

  for obj in json_arr:
    if set(obj.keys()) == set(string_to_search.split('-')):
      return obj

# desc  -> returns an array of string combination, of each objects keys
# json_arr          -> array of json objects
# returns           -> strings array
def extract_message_keys(json_arr):
  tag_arr = []
  for obj in json_arr:
    obj_keys = []
    for key in obj.keys():
      obj_keys.append(key)
    tag_arr.append('-'.join(sorted(obj_keys)))

  return tag_arr
                                                            ### utility functions end ###

                                                                ### tree generation ###
def tree_generation(json_arr):
  #intialize the tree
  tree_head = None

  # Initialize the array of tag strings and the dictionary of key-value pairs
  tag_arr = extract_message_keys(json_arr)

  messages_table = json_arr.copy()
  #calculate the distance table
  table_size = len(tag_arr)
  distances = distance_matrix_calculation(tag_arr, table_size)

  #tree construction loop when there is more than one tag
  while (table_size != 1):

    #find closest pair and coords of it in the table
    min_value = float('inf')
    min_row = 0
    min_col = 0
    for i in range(len(distances)):
      for j in range(i + 1, len(distances[i])):
        if distances[i][j] < min_value:
          min_value = distances[i][j]
          min_row = i
          min_col = j

    #get the closest messages from the messages array
    msg_1 = message_return(messages_table,tag_arr[min_col])
    msg_2 = message_return(messages_table,tag_arr[min_row])

    #combine the two msg
    aggregated_msgs = message_merge(msg_1,msg_2)

    #create the tree nodes that will be appended to the tree
    parent_of_childs = Node(aggregated_msgs)
    parent_of_childs.left = Node(msg_1)
    parent_of_childs.right = Node(msg_2)

    #check if one of the msg is the root of the tree
    first_tag_is_root = tree_head is not None and msg_1 == tree_head.msg
    second_tag_is_root = tree_head is not None and msg_2 == tree_head.msg

    if tree_head is None:
      #empty tree insert the parent as root
      tree_head = parent_of_childs

    elif first_tag_is_root and second_tag_is_root:
      #edge case
      pass

    elif first_tag_is_root:
      #if the tag_arr[min_col] is the current root.msg set futures root left child to tag_arr[min_row]
      #and set the future root child to the current root of the tree
      insert_left(parent_of_childs, msg_2)
      insert_parent_right(parent_of_childs, tree_head)
      tree_head = parent_of_childs

    elif second_tag_is_root:
      #if the tag_arr[min_row] is the current root.msg set futures root left child to tag_arr[min_col]
      #and set the future root child to the current root of the tree
      insert_left(parent_of_childs, msg_1)
      insert_parent_right(parent_of_childs, tree_head)
      tree_head = parent_of_childs

    else:
      #if none of the two msg are the root.msg then create a new sub tree with the closest pair
      insert_left(parent_of_childs, msg_1)
      insert_right(parent_of_childs, msg_2)

      #create a new super parent that will be the combination of the closest pair messages
      super_parent_of_childs = Node(message_merge(parent_of_childs.msg,tree_head.msg))
      #set the subtree root and the tree_head root as the children of the super parent root
      super_parent_of_childs.left = parent_of_childs
      super_parent_of_childs.right = tree_head
      #set the current tree root as the super parent root
      tree_head = super_parent_of_childs

    #pop the msg from the array and insert their combination
    messages_table.remove(msg_1)
    messages_table.remove(msg_2)
    messages_table.append(aggregated_msgs)

    #decrease the table_size cause of the msg combination
    table_size -= 1

    #update distances table with the distances of the new tag_arr
    tag_arr = extract_message_keys(messages_table)
    distances = distance_matrix_calculation(tag_arr, table_size)

  #print the levels of the tree
  level_order_traversal(tree_head)

#messages table
json_arr = [{
  "TEMP": 25,
  "HUM": 50
}, {
  "TEMP": 30,
  "CO2": 40,
  "HUM": 59
},{
  "H2O":54,
  "CO2": 40
}]

#tree generation function
tree_generation(json_arr)


import os, sys
import torch
import speechbrain as sb
from hyperpyyaml import load_hyperpyyaml
from thop import profile, clever_format

from train import Separation

hparam_file_list = [
    "baselines/tcn/convtasnet-whamr.yaml",
    "deformable/convtasnet-whamr.yaml",
    "deformable/shared_weights/convtasnet-whamr.yaml",
]

model_names = ["_".join(f.split("/")[:-1]) for f in hparam_file_list]
# print(model_names);exit()

run_opts_list = ["--data_folder /fastdata/acp19jwr/data/mono-whamr"]*len(hparam_file_list)

overrides_list = [""]*len(hparam_file_list)

sig_len=6

for h,r,o,m in zip(hparam_file_list,run_opts_list,overrides_list,model_names):
    hparams_file, run_opts, overrides = sb.parse_arguments([h]+r.split(" ")+o.split(" "))
    # print(sb.parse_arguments([h]+r.split(" ")+o.split(" ")));exit()

    with open(os.path.join("hparams",hparams_file)) as fin:
        hparams = load_hyperpyyaml(fin, overrides)

    run_opts["auto_mix_prec"] = hparams["auto_mix_prec"]

    input = torch.randn(1, hparams["sample_rate"]*sig_len).cuda()

    model = Separation(
            modules=hparams["modules"],
            opt_class=hparams["optimizer"],
            hparams=hparams,
            run_opts=run_opts,
            checkpointer=hparams["checkpointer"],
        )

    model.hparams.encode_rirs = False

    prof_macs, prof_params = model.compute_forward(input, None, sb.Stage.TEST, None, profiler=True)
    prof_macs["decoder"] = prof_macs["decoder"]*model.hparams.num_spks
    total_macs = sum(prof_macs.values())/sig_len
    total_params = sum(prof_params.values())
    macs, params = clever_format([total_macs, total_params], "%.3f")
    print(prof_macs["decoder"])
    print(m,macs,params)


import os

from fabric.api import env, execute, lcd, local, parallel, roles, task
from fabdeploytools import helpers
import fabdeploytools.envs

import deploysettings as settings

env.key_filename = settings.SSH_KEY
fabdeploytools.envs.loadenv(os.path.join('/etc/deploytools/envs',
                                         settings.CLUSTER))
MARKETPLACE_STATS = os.path.dirname(__file__)
ROOT = os.path.dirname(MARKETPLACE_STATS)

@task
def pre_update(ref):
    with lcd(MARKETPLACE_STATS):
        local('git fetch')
        local('git fetch -t')
        local('git reset --hard %s' % ref)

@task
def update():
    with lcd(MARKETPLACE_STATS):
        local('npm install')
        local('make install')
        local('cp src/media/js/settings_local_hosted.js src/media/js/settings_local.js')
        local('make build')
        local('node_modules/.bin/commonplace langpacks')

@task
def build():
    with lcd(MARKETPLACE_STATS):
        local('npm install')
        local('make install')
        local('cp src/media/js/settings_local_hosted.js '
              'src/media/js/settings_local.js')
        local('make build')
        local('node_modules/.bin/commonplace langpacks')

@task
def deploy_jenkins():
    rpmbuild = helpers.build_rpm(name='marketplace-stats',
                                 env=settings.ENV,
                                 cluster=settings.CLUSTER,
                                 domain=settings.DOMAIN,
                                 root=ROOT,
                                 package_dirs=['marketplace-stats'])

    rpmbuild.local_install()
    rpmbuild.remote_install(['web'])

@task
@roles('web')
@parallel
def _install_package(rpmbuild):
    rpmbuild.install_package()

@task
def deploy():
    with lcd(MARKETPLACE_STATS):
        ref = local('git rev-parse HEAD', capture=True)

    rpmbuild = helpers.deploy(name='marketplace-stats',
                              env=settings.ENV,
                              cluster=settings.CLUSTER,
                              domain=settings.DOMAIN,
                              root=ROOT,
                              deploy_roles=['web'],
                              package_dirs=['marketplace-stats'])


from predict import *

def evaluate(result, summary = False):

    avg = defaultdict(float) # average
    tp = defaultdict(int) # true positives
    tpfn = defaultdict(int) # true positives + false negatives
    tpfp = defaultdict(int) # true positives + false positives

    for _, y0, y1 in result: # actual value, prediction

        # evaluate on a token basis
        for y in y0:
            tp[y] += (y in y1)
            tpfn[y] += 1
        for y in y1:
            tpfp[y] += 1
        '''
        # evaluate on a sequence basis
        y0 = tuple(y0)
        y1 = tuple(y1)
        tp[y0] += (y0 == y1)
        tpfn[y0] += 1
        tpfp[y1] += 1
        '''

    for y in sorted(tpfn.keys()):

        pr = (tp[y] / tpfp[y]) if tpfp[y] else 0
        rc = (tp[y] / tpfn[y]) if tpfn[y] else 0
        avg["macro_pr"] += pr
        avg["macro_rc"] += rc

        if not summary:
            print("label =", y)
            print("precision = %f (%d/%d)" % (pr, tp[y], tpfp[y]))
            print("recall = %f (%d/%d)" % (rc, tp[y], tpfn[y]))
            print("f1 = %f" % f1(pr, rc))
            print()

    avg["macro_pr"] /= len(tpfn)
    avg["macro_rc"] /= len(tpfn)
    avg["micro_f1"] = sum(tp.values()) / sum(tpfn.values())

    print("macro precision = %f" % avg["macro_pr"])
    print("macro recall = %f" % avg["macro_rc"])
    print("macro f1 = %f" % f1(avg["macro_pr"], avg["macro_rc"]))
    print("micro f1 = %f" % avg["micro_f1"])

if __name__ == "__main__":

    if len(sys.argv) != 5:
        sys.exit("Usage: %s model char_to_idx word_to_idx test_data" % sys.argv[0])

    evaluate(predict(*load_model(), sys.argv[4]))


#!/usr/bin/env python

import itertools
import re
from collections import defaultdict

class Grammar(dict):
    """
    """
    def __init__(self, training, sigma=None, max_k=2):
        """
        Attributes:
        lexicon
        sigma
        max_k
        tiers
        grammatical
        ungrammatical
        """
        self.read_training_file(training)
        self.get_sigma(sigma)
        self.max_k = max_k
        self.build_tiers()

    def learn(self):
        """
        """
        self.grammatical = {}
        self.ungrammatical = {}
        for tier in self.tiers:
            tier_grammatical = []
            possible_ngrams = set(itertools.chain.from_iterable(itertools.product(tier, repeat=r) for r in range(1,self.max_k+1)))
            for form in self.lexicon:
                on_tier = [s for s in form if s in tier]
                # print(tier)
                # print(form)
                # print(on_tier)
                # print()
                tier_grammatical += self.get_ngrams(on_tier, self.max_k)
            grammatical_set = set(tier_grammatical)
            ungrammatical_set = possible_ngrams - grammatical_set
            self.grammatical[tuple(tier)] = grammatical_set
            self.ungrammatical[tuple(tier)] = ungrammatical_set

    def read_training_file(self, training_file):
        with open(training_file) as tf:
            lines = tf.read().rstrip().replace('\r','\n').replace('\n\n','\n').split('\n')
            self.lexicon = [line.split(' ') for line in lines]

    def get_sigma(self, sigma_file):
        if sigma_file == None:
            # infer from self.training
            merged_data = list(itertools.chain.from_iterable(self.lexicon))
            self.sigma = list(set(merged_data))
        else:
            # TO-DO: read file
            pass

    def build_tiers(self):
        """All possible tiers
        Each tier is a tuple in alphabetical order
        """
        self.tiers = []
        for size in range(1,len(self.sigma)+1):
            for combo in itertools.combinations(self.sigma, size):
                self.tiers.append(tuple(sorted(list(combo))))

    def distill_grammar(self):
        grammar = defaultdict(list)
        for kfactor in itertools.permutations(self.sigma, 2): # could this consider instead only bigrams prohibited on the segment tier?
            relevant_tier = self.traverse_lattice(kfactor)
            if relevant_tier != None:
                grammar[relevant_tier].append(kfactor)
        return grammar

    def traverse_lattice(self, kfactor):
        """
        """
        initial_tier = set(kfactor)
        not_in_kfactor = set(self.sigma) - initial_tier

        initial_tier_tuple = tuple(sorted(kfactor))
        if kfactor in self.ungrammatical[initial_tier_tuple]:
            return initial_tier_tuple

        result = initial_tier.copy()
        for char in not_in_kfactor:
            combined_tier = initial_tier.union(set([char]))
            combined_tier_tuple = tuple(sorted(list(combined_tier)))
            if kfactor in self.ungrammatical[combined_tier_tuple]:
                result.add(char)

        if result == initial_tier:
            full_tier = tuple(sorted(self.sigma))
            if kfactor in self.ungrammatical[full_tier]:
                return full_tier
            else:
                return None
        else:
            return tuple(sorted(list(result)))

    def tiers_equal(tier1, tier2):
        return set(tier1) == set(tier2)

    def get_ngrams(self, sequence, max_k):
        return list(itertools.chain.from_iterable([zip(*[sequence[i:] for i in range(n)]) for n in range(max_k+1)]))

if __name__ == '__main__':
    g = Grammar('cvlrb6_blocking.txt')

    g.learn()

    # for tier in g.grammatical:
    #     print('Tier: {}'.format(str(tier)))
    #     print('Grammatical:')
    #     print(g.grammatical[tier])
    #     print('Ungrammatical:')
    #     print(g.ungrammatical[tier])
    #     print()

    dg = g.distill_grammar()
    for tier in dg:
        if len(dg[tier]) > 0:
            print('Prohibited on tier {}:'.format(str(tuple(dg)[0])))
            for seq in dg[tier]:
                print(seq)


"""
	Data preprocessing and cleaning as seen in previous works. Full pipeline is broken down into small functions in order to make it modular and easier to change.
"""

import pandas as pd
import numpy as np

def print_info(df:pd.DataFrame):
	"""
		Prints some usefull information
	"""
	print('-'*30)
	print(df.info())
	print(df.describe())

def clean(df:pd.DataFrame) -> pd.DataFrame:
	"""
		Performs basic cleaning to database
	"""
	# Remove duplicate rows (there are usually no duplicates, but jic)
	df = df.drop_duplicates()
	# Check 0<fuel_level<100
	df.fuel_level[(df.fuel_level<0) | (df.fuel_level>100)] = np.nan
	return df

# Clean plate strings (they must all be a combination of letters and numbers without spaces so we eliminate them
# and everything that goes after them)

# This is designed ad hoc for some errors that are of the type: 5498LFN ITER FRUGAL
# COMMENTED: DO NOT CLEAN PLATES AS THEY ARE IDs DEFINED BY THE CLIENTS
# df.plate = df.plate.str[:7]

# TO DO: Check if acumulative data is indeed acumulative. This code block is almost the same as Heng's (incomplete)
# ac = ["engine_hours", "total_consum", "accelerations", "breaks", "clutch", "slowmotion_time"]
# dfac = df[ac]
# same = [df.iloc[i].plate == df.iloc[i-1].plate for i in df.index]
# same

def split_fractal_into_columns(df:pd.DataFrame) -> pd.DataFrame:
	"""
		Name is self-explainatory
	"""
	# Split fractal into different strings
	add = df.fractal.str.split(',', expand=True)
	add.describe()

	# Columns 2 and 3 are always empty
	add = add.drop(columns=[2, 3])

	# Assign split values and drop original
	df["street"] = add[0]
	df["postal_code"] = add[1]
	df["city"] = add[4]
	df["province"] = add[5]
	df["state"] = add[6]
	df["country"] = add[7]

	return df.drop(columns=["fractal"])

def preprocessing(df:pd.DataFrame) -> pd.DataFrame:
	"""
		Execute every function in this module
	"""
	df = clean(df)
	df = split_fractal_into_columns(df)
	print_info(df)
	return df


import matplotlib.pyplot as plt
from matplotlib.colors import LinearSegmentedColormap

from grid import *
import matplotlib.cm as cm
import pandas as pd

answers = pd.read_csv("answers.csv")
answers = pd.DataFrame([answers.iloc[:, 2], answers.iloc[:, 3]]).T
answers.columns = ["segment", "zone"]
answers["bet"] = answers["segment"] + answers["zone"].map(str)
values = answers.bet.value_counts().values

color_map = cm.get_cmap("Oranges")

for ii, radius in enumerate(radii):
    if ii == 0:
        value = (answers["segment"] == "A").sum()
        print(f"A: {value}")
        value_normalized = value / (values.max())
        color = color_map(value_normalized)
        circle = Circle(
            center_coords,
            radius=radius,
            edgecolor="none",
            facecolor=color,
            alpha=0.5,
        )
        ax.add_patch(circle)
        continue
    zone_angles = zone_angles_list[ii - 1]
    zone_angles = np.append(zone_angles, zone_angles[0] + 2 * np.pi)  # close the loop
    number_of_zones = number_of_zones_list[ii - 1]
    for jj in range(int(number_of_zones)):
        theta_range = np.linspace(zone_angles[jj], zone_angles[jj + 1], 100)
        r_range = np.linspace(radii[ii - 1], radius, 100)
        x_inner = center_x + radii[ii - 1] * np.cos(theta_range)
        y_inner = center_y + radii[ii - 1] * np.sin(theta_range)
        x_outer = center_x + radius * np.cos(theta_range)
        y_outer = center_y + radius * np.sin(theta_range)
        x_range = np.append(x_inner, x_outer[::-1])
        y_range = np.append(y_inner, y_outer[::-1])
        bet = f"{alphabet[ii]}{jj}"
        value = (answers["bet"] == bet).sum()
        print(f"{bet}: {value}")
        value_normalized = value / (values.max())
        color = color_map(value_normalized)
        if value > 0:
            ax.fill(x_range, y_range, color=color, alpha=0.5, zorder=0)

fig.set_size_inches(img.size[0] / 100, img.size[1] / 100 + 2.7)
fig.subplots_adjust(top=1, bottom=1 - img.size[1] / 100 / (img.size[1] / 100 + 2.7), left=0, right=1)

# Setup colorbar
color_map_list = np.array([color_map(i) for i in range(color_map.N)])
color_map_list[:, -1] = 0.7
custom_map = LinearSegmentedColormap.from_list(
    'Custom cmap', color_map_list.tolist(), color_map.N)
bounds = np.linspace(1, values.max() + 1, values.max() + 1)
sm = cm.ScalarMappable(cmap=custom_map, norm=plt.Normalize(vmin=0, vmax=values.max()))
sm.set_array([])
cax = ax.inset_axes([0.05, -0.07, 0.9, 0.045])
cbar = fig.colorbar(sm, orientation="horizontal", cax=cax, boundaries=bounds,
                    ticks=np.arange(0.5, values.max() + 1.5))
cbar.ax.set_xticklabels(np.arange(0, values.max() + 1))
cbar.ax.xaxis.set_tick_params(pad=30)
cbar.ax.set_facecolor('none')
cbar.ax.tick_params(length=0)
cbar.outline.set_edgecolor('white')
cbar.outline.set_linewidth(5)
plt.setp(plt.getp(cbar.ax, 'xticklabels'), color='white', fontsize=50,)

plt.savefig("heatmap.png", facecolor="None")
plt.savefig("heatmap_black.png")
plt.show()


from graphics.Geometry import Line
from graphics.Vector import Vector as vec2
from graphics.Helpers import map
import os
import socket
import sys
sys.path.append("..")
import ob_helper
import random
import math

# Some variables
width, height = 1000, 1000

# Main function
def draw():

    border = 50
    border_sQ = border*2
    num_lines = 8
    y_step = float((height-border_sQ)/num_lines)
    x_step = float((width-border_sQ)/num_lines)
    y_offset = y_step/2.0
    x_offset = x_step/2.0
    un_offset = 10
    my_lines = []

    for i in range(num_lines):
        for j in range(num_lines-1):
            if j == 0:
                p1 = vec2([
                    x_step*j+x_offset + random.uniform(-x_offset, x_offset) + border,
                    y_step*i+y_offset + random.uniform(-y_offset, y_offset) + border
                ])
                p2 = vec2([
                    x_step*(j+1)+x_offset + random.uniform(-x_offset, x_offset) + border,
                    y_step*i+y_offset + random.uniform(-y_offset, y_offset) + border
                ])
                my_lines.append(Line(p1[0], p1[1], p2[0], p2[1]))
            else:
                p1 = vec2([
                    my_lines[(j+(num_lines-1)*i)-1].p1[0],
                    my_lines[(j+(num_lines-1)*i)-1].p1[1]
                ])
                p2 = vec2([
                    x_step*(j+1)+x_offset + random.uniform(-x_offset, x_offset) + border,
                    y_step*i+y_offset + random.uniform(-y_offset, y_offset) + border
                ])
                my_lines.append(Line(p1[0], p1[1], p2[0], p2[1]))

    index = 0
    for i in range(num_lines):
        index = index + 1
        for j in range(num_lines-1):
            if i != 0:
                lerp_lines = int(map(i, 0, num_lines, 1, 12))+1
                for k in range(lerp_lines):
                    p0 = my_lines[(j+(num_lines-1)*(i-1))].get_lerp(math.pow(map(k, 0, lerp_lines-1, 0, 1), 1))
                    p1 = my_lines[(j+(num_lines-1)*i)].get_lerp(math.pow(map(k, 0, lerp_lines-1, 0, 1), 1))
                    ob_helper.drawLineOB(p0,p1)

    # CHANGE COLOR FOR LERP LINES (THE INNER BORDERS)
    randomColor = format(random.randint(0,16777215),'x')
    ob_helper.sendCommands([f"color.set.html={randomColor}"])
    for line in my_lines:
        ob_helper.drawLineOB(line.p0,line.p1)

"""
  curl -v "http://10.0.2.2:40074/api/v1?new"
  curl -v "http://10.0.2.2:40074/api/v1?user.move.to=5,10,10"
  curl -v "http://10.0.2.2:40074/api/v1?brush.size.set=0.5"
  curl -v "http://10.0.2.2:40074/api/v1?brush.type=Icing"

randomColor="$(openssl rand -hex 3)"; curl -v "http://10.0.2.2:40074/api/v1?color.set.html=$randomColor"
"""
def main():
    hostname = socket.gethostname()
    if "OB_HOST" in os.environ:
      ob_helper.ob_host=os.environ['OB_HOST']

    ob_helper.sendCommands(["new"])
    ob_helper.sendCommands(["brush.move.to=0,0,0","brush.look.up"])
    ob_helper.sendCommands(["user.move.to=-5,10,10"])

    # ob_helper.sendCommands(["brush.type=Icing"])
    # ob_helper.sendCommands(["brush.size.set=0.4"])

    # ob_helper.sendCommands(["brush.type=NeonPulse"])
    # ob_helper.sendCommands(["brush.size.set=1"])

    ob_helper.sendCommands(["brush.type=Light"])
    ob_helper.sendCommands(["brush.size.set=.4"])

    # ob_helper.sendCommands(["brush.type=Electricity"])
    # ob_helper.sendCommands(["brush.size.set=.1"])

    draw()
    ob_helper.sendCommands(["debug.brush"])

# Call the main function
if __name__ == '__main__':
    main()


from datetime import datetime, timedelta
from django import template

register = template.Library()

class serialno():
    def __init__(self):
        self.sno = 0

s = serialno()

@register.filter
def is_date_greater_than_one_day(value):
    value_date = datetime.strptime(value, '%Y-%m-%d')
    one_day_later = datetime.now() + timedelta(days=1)
    return value_date > one_day_later

@register.filter
def inc(value):
    s.sno += 1
    return s.sno

@register.filter
def reset(value):
    s.sno = 0
    return s.sno


import torch.nn as nn
import torch.nn.functional as F
import torch
# from archs.common import make_layer
from utils.registry import ARCH_REGISTRY
from archs.convLSTM import ConvLSTMCell
import numpy as np
from archs.Net_Big import Diff_Enc   # 特征生成

class HDA_conv(nn.Module):   # SDL
    def __init__(self, chan_fixed, chan_dyn, num_feat, kernel_size, attn=False):
        super(HDA_conv, self).__init__()

        self.kernel_size = kernel_size   # 卷积核大小
        # self.attn = attn
        self.num_feat = num_feat   # ？

        # 固定的参数部分  从图像中直接提取出来的特征
        # 输入图像的四维张量[N, C, H, W]
        # Conv2d(in_channels,out_channels,kernel_size,stride=1,padding=0,dilation=1,groups=1,bias=True,padding_mode='zeros')
        # stride（移动的长度） padding（填充与否）groups（是否用分组卷积）
        self.conv_share = nn.Conv2d(num_feat, chan_fixed, kernel_size, stride=1, padding=(kernel_size-1)//2)    # (kernel_size-1)//2 ？
        # 需要修改的参数部分
        self.conv_dyn = conv_dyn(num_feat, chan_dyn, kernel_size, stride=1, padding=(kernel_size-1)//2, bias=False)
        self.relu = nn.LeakyReLU(0.1, True)

    def forward(self, x, out_weight):
        # calculate conv weights
        # out_weight size: [chan_dyn, num_feat, kernel_size, kernel_size]

        # b, c, h, w = x.size()

        out1 = self.conv_dyn(x, out_weight)
        out2 = self.conv_share(x)

        out = torch.cat((out1, out2), dim=1)  # dim维度 在横向叠加  SDL

        return out

# class conv_dyn(nn.Module):
#
#     def forward(self, input, weight):
#         # weight size: [b*out, in ,k, k]
#         b, c, h, w = input.size()
#         out = F.conv2d(input.view(1, -1, h, w), weight, stride=1, padding=(weight.size(2)-1)//2, groups=b*c)
#         return out.view(b, -1, h, w)

class conv_dyn(nn.Conv2d):   # 如何对需要生成的参数  进行卷积

    def _conv_forward(self, input, weight, bias, groups):

        if self.padding_mode != 'zeros':
            # F.pad(input, pad, mode='constant', value=0) 对tensor进行扩充  input:需要扩充的tensor，pad：扩充维度，mode：扩充方法 value：扩充时指定补充值
            return F.conv2d(F.pad(input, self._reversed_padding_repeated_twice, mode=self.padding_mode),
                            weight, bias, self.stride,
                            (0, 0), self.dilation, groups)
        return F.conv2d(input, weight, bias, self.stride,
                        self.padding, self.dilation, groups)

    def forward(self, input, weight):
        # weight size: [b*out, in ,k, k]
        b, c, h, w = input.size()
        conv_weight = self.weight.repeat(b, 1,1,1)   # repeat()给三个维度各加一行
        out = self._conv_forward(input.view(1, -1, h, w), conv_weight * weight, self.bias, groups=b)    # 最后的输出结果
        return out.view(b, -1, h, w)  # 将一维变为b维

class ResidualBlockNoBN_dyn(nn.Module):  # 系统图中的灰色部分 残差块
    """Residual block without BN.
    It has a style of:
    ::
        ---Conv-ReLU-Conv-+-
         |________________|
    Args:
        mid_channels (int): Channel number of intermediate features.
            Default: 64.
        res_scale (float): Used to scale the residual before addition.
            Default: 1.0.
    """

    def __init__(self, chan_fixed, chan_dyn, num_feat, kernel_size=3, attn=False, res_scale=1.0):
        super(ResidualBlockNoBN_dyn, self).__init__()
        self.res_scale = res_scale
        # 代表灰色的那一块的两个SDLs        先卷积再cat
        self.conv1 = HDA_conv(chan_fixed, chan_dyn, num_feat, kernel_size, attn)
        self.conv2 = HDA_conv(chan_fixed, chan_dyn, num_feat, kernel_size, attn)

        # 激活函数
        self.relu = nn.ReLU(inplace=True)

    def forward(self, x, out_weight1, out_weight2):

        identity = x
        out = self.conv1(x, out_weight1)
        # out = self.relu(self.bn1(out))
        out = self.relu(out)
        out = self.conv2(out, out_weight2)
        out = identity + out * self.res_scale
        # out = self.relu(out)
        return out

@ARCH_REGISTRY.register()
class DynEDSR(nn.Module):    # 主架构  定义模型
    def __init__(self,
                 num_in_ch,
                 num_feat=64,
                 num_block=16,
                 res_scale=1,          # ？
                 img_range=255.,
                 rgb_mean=(0.4488, 0.4371, 0.4040),
                 kernel_size=3,
                 weight_ratio=0.5,
                 attn=False,
                 blind=False,
                 ):

        super(DynEDSR, self).__init__()

        # 下面就一层层开始定义需要用到的函数及方法
        self.img_range = img_range    # 图像范围
        self.mean = torch.Tensor(rgb_mean).view(1, 3, 1, 1)   # 归一化？
        self.num_feat = num_feat
        self.conv_first = nn.Conv2d(num_in_ch, num_feat, 3, 1, 1)   # 第一个卷积 将输入通道数变为num_feat
        self.kernel_size = kernel_size    # 卷积核大小
        chan_fixed = int(num_feat * weight_ratio)    # 一半固定的参数
        chan_dyn = num_feat - chan_fixed             # 一半需要改变的参数
        self.attn = attn
        self.relu = nn.LeakyReLU(0.1, True)    # 激活函数
        self.blind = blind

        if attn:   # attn非零
            if blind:   # blind非零
                self.embed = Diff_Enc()
                # 加载模型  checkpoint：通用的检查点
                checkpoint = torch.load('./experiments/feature_extract.pth')
                self.embed.load_state_dict(checkpoint)

                # self.embed = Degrade_Encoder()
                # checkpoint = torch.load('/home/lhm/PycharmProject/GIR/experiments/FeatEncoder_v1_96/models/net_g_180000.pth')
                # self.embed.load_state_dict(checkpoint['params'])

                self.conv_represent = nn.Sequential(  # 全连接层 DEM之后的那个全连接层
                    nn.Linear(256, num_feat, bias=False),
                    nn.LeakyReLU(0.1, True),
                    nn.Linear(num_feat, chan_dyn * num_feat * kernel_size * kernel_size, bias=False))
            else:

                self.conv_represent = nn.Sequential(   # 计算全连接层的  DAM的输入
                    nn.Linear(3, num_feat, bias=False),
                    nn.LeakyReLU(0.1, True),
                    # nn.Linear(num_feat, chan_dyn*num_feat, bias=False),
                    # nn.LeakyReLU(0.1, True),
                    nn.Linear(num_feat, chan_dyn * num_feat * kernel_size * kernel_size, bias=False)
                )

            # self.forward_lstm = nn.ModuleList()
            # self.backward_lstm = nn.ModuleList()
            # for _ in range(num_block*2):
            #     self.forward_lstm.append(
            #         ConvLSTMCell((kernel_size, kernel_size), num_feat, num_feat, 1, bias=False)
            #     )
            #
            #     self.backward_lstm.append(
            #         ConvLSTMCell((kernel_size, kernel_size), num_feat, num_feat, 1, bias=False)
            #     )
            # LSTM的过程
            self.forward_lstm = ConvLSTMCell((kernel_size, kernel_size), num_feat, num_feat, 1, bias=False)   # 前向
            self.backward_lstm = ConvLSTMCell((kernel_size, kernel_size), num_feat, num_feat, 1, bias=False)  # 后向
            self.lstm_conv_list = nn.ModuleList()
            for _ in range(num_block*2):    # 前后向之后的卷积  前项和后向各来一遍  就得是num_block*2
                self.lstm_conv_list.append(   # 将Conv2d添加进ModuleList列表中
                    nn.Conv2d(num_feat * 2, num_feat, 1, 1, 0)
                )

        self.num_block = num_block
        self.body = nn.ModuleList()
        for _ in range(num_block):
            self.body.append(
                ResidualBlockNoBN_dyn(   # 一个灰色（HDA）的输出
                    chan_fixed=chan_fixed,
                    chan_dyn=chan_dyn,
                    num_feat=num_feat,
                    kernel_size=kernel_size,
                    attn=attn,
                    res_scale=res_scale
                ))

        self.conv_after_body = nn.Conv2d(num_feat, num_feat, 3, 1, 1)
        self.conv_last = nn.Conv2d(num_feat, num_in_ch, 3, 1, 1)       # 变为与最原始的输入图像的channel一样

    def forward(self, x, cond):

        # cond : b, 3
        # b, c, h, w = x.size()

        if self.attn:
            if self.blind:
                # with torch.no_grad():
                if cond.size(1) == 64:
                    represent = cond
                else:
                    # self.embed.eval()
                    # print('realblur')
                    represent, _ = self.embed(x)
                    # _, represent = self.embed(x)

                represent = self.conv_represent(represent)  # b, chan_dyn*num_feat*3*3
                represent = represent.view(-1, self.num_feat, self.kernel_size,
                                           self.kernel_size)  # b*chan_dyn, num_feat, kernel_size, kernel_size
            else:
                represent = self.conv_represent(cond)  # b, chan_dyn*num_feat*3*3
                represent = represent.view(-1, self.num_feat, self.kernel_size,
                                           self.kernel_size)  # b*chan_dyn, num_feat, kernel_size, kernel_size

            h_init_for = torch.zeros_like(represent)
            c_init_for = torch.zeros_like(represent)
            next_state_for = (h_init_for, c_init_for)

            h_init_back = torch.zeros_like(represent)
            c_init_back = torch.zeros_like(represent)
            next_state_back = (h_init_back, c_init_back)

            out_weight_for = []
            out_weight_back = []

            for i in range(self.num_block*2):
                next_state_for = self.forward_lstm(represent, next_state_for)
                out_weight_for.append(next_state_for[0])
                next_state_back = self.backward_lstm(represent, next_state_back)
                out_weight_back.append(next_state_back[0])

            out_weight_back.reverse()

        self.mean = self.mean.type_as(x)
        x = (x - self.mean) * self.img_range
        x = self.conv_first(x)

        x1 = x.clone()    # x=x1
        for j in range(self.num_block):
            if self.attn:
                out_weight1 = self.lstm_conv_list[j*2](torch.cat((out_weight_for[j*2], out_weight_back[j*2]), dim=1))
                out_weight2 = self.lstm_conv_list[j*2+1](torch.cat((out_weight_for[j*2+1], out_weight_back[j*2+1]), dim=1))

            else:
                out_weight1 = 1
                out_weight2 = 1

            x1 = self.body[j](x1, out_weight1, out_weight2)

        res = self.conv_after_body(x1)
        res += x
        x = self.conv_last(res)
        x = x / self.img_range + self.mean
        return x


#!/usr/bin/env python3.7
# Description:
# - Subscriber to convert ROS Image to OpenCV
#
# Author:
# - Kwasi

import rospy
from sensor_msgs.msg import Image
from cv_bridge import CvBridge
import cv2

def callback(data):

    # Used to convert between ROS and OpenCV images
    br = CvBridge()

    # Output debugging information to the terminal
    rospy.loginfo("receiving video frame")

    # Convert ROS Image message to OpenCV image
    current_frame = br.imgmsg_to_cv2(data)

    # Display image
    cv2.imshow("camera", current_frame)

    cv2.waitKey(1)

def receive_message():
    # Initialize ROS
    rospy.init_node('camera_subscriber', anonymous=True)

    # Node is subscribing to the video_frames topic
    rospy.Subscriber('/prymbot/image_raw', Image, callback)

    # spin() simply keeps python from exiting until this node is stopped
    rospy.spin()

    # Close down the video stream when done
    cv2.destroyAllWindows()

if __name__ == '__main__':
    receive_message()


import os
import sys
sys.path.insert(0, '..')
import torch
from torch import optim, nn, utils, Tensor
from torchvision.datasets import MNIST
from torchvision.transforms import ToTensor
import pytorch_lightning as pl

from python_environment_check import check_packages

d = {
    'torch': '1.8',
    'torchvision': '0.9.0',
    'tensorboard': '2.7.0',
    'pytorch_lightning': '1.5.0',
    'torchmetrics': '0.6.2'
}
check_packages(d)

"""
Define a LightningModule

A LightningModule enables your PyTorch nn.Module to play together in complex
ways inside the training_step (there is also an optional validation_step and
test_step).
"""

# define any number of nn.Modules (or use your current ones)
encoder = nn.Sequential(nn.Linear(28 * 28, 64), nn.ReLU(), nn.Linear(64, 3))
decoder = nn.Sequential(nn.Linear(3, 64), nn.ReLU(), nn.Linear(64, 28 * 28))

# define the LightningModule
class LitAutoEncoder(pl.LightningModule):
    def __init__(self, encoder, decoder):
        super().__init__()
        self.encoder = encoder
        self.decoder = decoder

    def training_step(self, batch, batch_idx):
        # training_step defines the train loop.
        # it is independent of forward
        x, y = batch
        x = x.view(x.size(0), -1)
        z = self.encoder(x)
        x_hat = self.decoder(z)
        loss = nn.functional.mse_loss(x_hat, x)
        # Logging to TensorBoard (if installed) by default
        self.log("train_loss", loss)
        return loss

    def test_step(self, batch, batch_idx):
        # this is the test loop
        x, y = batch
        x = x.view(x.size(0), -1)
        z = self.encoder(x)
        x_hat = self.decoder(z)
        test_loss = F.mse_loss(x_hat, x)
        self.log("test_loss", test_loss)

    def configure_optimizers(self):
        optimizer = optim.Adam(self.parameters(), lr=1e-3)
        return optimizer

# init the autoencoder
autoencoder = LitAutoEncoder(encoder, decoder)

"""
Define a dataset

Lightning supports ANY iterable (DataLoader, numpy, etc…) for the
train/val/test/predict splits.
"""

# setup data
dataset = MNIST(os.getcwd(), download=True, transform=ToTensor())
train_loader = utils.data.DataLoader(dataset)

"""
Train the model

The Lightning Trainer “mixes” any LightningModule with any dataset and abstracts
away all the engineering complexity needed for scale.

See documentation in lightning.ai
"""

# train the model (hint: here are some helpful Trainer arguments for rapid idea iteration)
trainer = Trainer()

stop

trainer = pl.Trainer(limit_train_batches=100, max_epochs=1)
trainer.fit(model=autoencoder, train_dataloaders=train_loader)

"""
Use the model

Once you’ve trained the model you can export to onnx, torchscript and put it
into production or simply load the weights and run predictions.
"""

# load checkpoint
checkpoint = "./lightning_logs/version_48/checkpoints/epoch=0-step=100.ckpt"
autoencoder = LitAutoEncoder.load_from_checkpoint(checkpoint, encoder=encoder, decoder=decoder)

# choose your trained nn.Module
encoder = autoencoder.encoder
encoder.eval()

# embed 4 fake images!
fake_image_batch = Tensor(4, 28 * 28)
embeddings = encoder(fake_image_batch)
print("⚡" * 20, "\nPredictions (4 image embeddings):\n", embeddings, "\n", "⚡" * 20)

"""
Visualize training

If you have tensorboard installed, you can use it for visualizing experiments.

Run this on your commandline and open your browser to http://localhost:6006/
tensorboard --logdir .

"""

"""
Supercharge training

Enable advanced training features using Trainer arguments. These are
state-of-the-art techniques that are automatically integrated into your training
loop without changes to your code.

# train on 4 GPUs
trainer = Trainer(
    devices=4,
    accelerator="gpu",
 )

train 1TB+ parameter models with Deepspeed/fsdp
trainer = Trainer(
    devices=4,
    accelerator="gpu",
    strategy="deepspeed_stage_2",
    precision=16
 )

# 20+ helpful flags for rapid idea iteration
trainer = Trainer(
    max_epochs=10,
    min_epochs=5,
    overfit_batches=1
 )

# access the latest state of the art techniques
trainer = Trainer(callbacks=[StochasticWeightAveraging(...)])
"""

import torchvision.transforms as transforms
import torch.utils.data as data
from torchvision import datasets

"""
Add a test loop

To make sure a model can generalize to an unseen dataset (ie: to publish a paper
or in a production environment) a dataset is normally split into two parts, the
train split and the test split.

The test set is NOT used during training, it is ONLY used once the model has
been trained to see how the model will do in the real-world.

Find the train and test splits

Datasets come with two splits. Refer to the dataset documentation to find the
train and test splits.
"""

# define any number of nn.Modules (or use your current ones)

# Load data sets
transform = transforms.ToTensor()
train_set = datasets.MNIST(root="MNIST", download=True, train=True, transform=transform)
test_set = datasets.MNIST(root="MNIST", download=True, train=False, transform=transform)

"""
Define the test loop

To add a test loop, implement the test_step method of the LightningModule
"""
"""
Train with the test loop

Once the model has finished training, call .test
"""

from torch.utils.data import DataLoader

# initialize the Trainer
trainer = Trainer()

# test the model
trainer.test(model, dataloaders=DataLoader(test_set))

"""
Add a validation loop

During training, it’s common practice to use a small portion of the train split
to determine when the model has finished training.
As a rule of thumb, we use 20% of the training set as the validation set.
This number varies from dataset to dataset.
"""

# Split the training data
# use 20% of training data for validation
train_set_size = int(len(train_set) * 0.8)
valid_set_size = len(train_set) - train_set_size

# split the train set into two
seed = torch.Generator().manual_seed(42)
train_set, valid_set = data.random_split(train_set, [train_set_size, valid_set_size], generator=seed)


"""Tools for downloading map tiles from coordinates."""
from __future__ import (absolute_import, division, print_function)

import mercantile as mt
import requests
import io
import os
import numpy as np
import rasterio as rio
from PIL import Image
from rasterio.transform import from_origin
from . import tile_providers as sources

__all__ = ['bounds2raster', 'bounds2img', 'howmany']

def bounds2raster(w, s, e, n, path, zoom='auto',
                  url=sources.ST_TERRAIN, ll=False,
                  wait=0, max_retries=2):
    '''
    Take bounding box and zoom, and write tiles into a raster file in
    the Spherical Mercator CRS (EPSG:3857)

    ...

    Arguments
    ---------
    w       : float
              West edge
    s       : float
              South edge
    e       : float
              East edge
    n       : float
              Noth edge
    zoom    : int
              Level of detail
    path    : str
              Path to raster file to be written
    url     : str
              [Optional. Default:
              'http://tile.stamen.com/terrain/tileZ/tileX/tileY.png'] URL for
              tile provider. The placeholders for the XYZ need to be `tileX`,
              `tileY`, `tileZ`, respectively. See `cx.sources`.
    ll      : Boolean
              [Optional. Default: False] If True, `w`, `s`, `e`, `n` are
              assumed to be lon/lat as opposed to Spherical Mercator.
    wait    : int
              [Optional. Default: 0]
              if the tile API is rate-limited, the number of seconds to wait
              between a failed request and the next try
    max_retries: int
                 [Optional. Default: 2]
                 total number of rejected requests allowed before contextily
                 will stop trying to fetch more tiles from a rate-limited API.

    Returns
    -------
    img     : ndarray
              Image as a 3D array of RGB values
    extent  : tuple
              Bounding box [minX, maxX, minY, maxY] of the returned image
    '''
    if not ll:
        # Convert w, s, e, n into lon/lat
        w, s = _sm2ll(w, s)
        e, n = _sm2ll(e, n)
    if zoom == 'auto':
        zoom = _calculate_zoom(w, s, e, n)
    # Download
    Z, ext = bounds2img(w, s, e, n, zoom=zoom, url=url, ll=True)
    # Write
    # ---
    h, w, b = Z.shape
    # --- https://mapbox.github.io/rasterio/quickstart.html#opening-a-dataset-in-writing-mode
    minX, maxX, minY, maxY = ext
    x = np.linspace(minX, maxX, w)
    y = np.linspace(minY, maxY, h)
    resX = (x[-1] - x[0]) / w
    resY = (y[-1] - y[0]) / h
    transform = from_origin(x[0] - resX / 2,
                            y[-1] + resY / 2, resX, resY)
    # ---
    raster = rio.open(path, 'w',
                      driver='GTiff', height=h, width=w,
                      count=b, dtype=str(Z.dtype.name),
                      crs='epsg:3857', transform=transform)
    for band in range(b):
        raster.write(Z[:, :, band], band + 1)
    raster.close()
    return Z, ext

def bounds2img(w, s, e, n, zoom='auto',
               url=sources.ST_TERRAIN, ll=False,
               wait=0, max_retries=2):
    '''
    Take bounding box and zoom and return an image with all the tiles
    that compose the map and its Spherical Mercator extent.

    ...

    Arguments
    ---------
    w       : float
              West edge
    s       : float
              South edge
    e       : float
              East edge
    n       : float
              Noth edge
    zoom    : int
              Level of detail
    url     : str
              [Optional. Default: 'http://tile.stamen.com/terrain/tileZ/tileX/tileY.png']
              URL for tile provider. The placeholders for the XYZ need to be
              `tileX`, `tileY`, `tileZ`, respectively. IMPORTANT: tiles are
              assumed to be in the Spherical Mercator projection (EPSG:3857).
    ll      : Boolean
              [Optional. Default: False] If True, `w`, `s`, `e`, `n` are
              assumed to be lon/lat as opposed to Spherical Mercator.
    wait    : int
              [Optional. Default: 0]
              if the tile API is rate-limited, the number of seconds to wait
              between a failed request and the next try
    max_retries: int
                 [Optional. Default: 2]
                 total number of rejected requests allowed before contextily
                 will stop trying to fetch more tiles from a rate-limited API.

    Returns
    -------
    img     : ndarray
              Image as a 3D array of RGB values
    extent  : tuple
              Bounding box [minX, maxX, minY, maxY] of the returned image
    '''
    if not ll:
        # Convert w, s, e, n into lon/lat
        w, s = _sm2ll(w, s)
        e, n = _sm2ll(e, n)
    if zoom == 'auto':
        zoom = _calculate_zoom(w, s, e, n)
    tiles = []
    arrays = []
    for t in mt.tiles(w, s, e, n, [zoom]):
        x, y, z = t.x, t.y, t.z
        tile_url = url.replace('tileX', str(x)).replace('tileY', str(y)).replace('tileZ', str(z))
        # ---
        image = _fetch_tile(tile_url, wait, max_retries)
        # ---
        tiles.append(t)
        arrays.append(image)
    merged, extent = _merge_tiles(tiles, arrays)
    # lon/lat extent --> Spheric Mercator
    west, south, east, north = extent
    left, bottom = mt.xy(west, south)
    right, top = mt.xy(east, north)
    extent = left, right, bottom, top
    return merged, extent

def _fetch_tile(tile_url, wait, max_retries):
    request = _retryer(tile_url, wait, max_retries)
    with io.BytesIO(request.content) as image_stream:
        image = Image.open(image_stream).convert('RGB')
        image = np.asarray(image)
    return image

def _retryer(tile_url, wait, max_retries):
    """
    Retry a url many times in attempt to get a tile

    Arguments
    ---------
    tile_url: str
              string that is the target of the web request. Should be
              a properly-formatted url for a tile provider.
    wait    : int
              if the tile API is rate-limited, the number of seconds to wait
              between a failed request and the next try
    max_retries: int
                 total number of rejected requests allowed before contextily
                 will stop trying to fetch more tiles from a rate-limited API.

    Returns
    -------
    request object containing the web response.
    """
    try:
        request = requests.get(tile_url)
        request.raise_for_status()
    except requests.HTTPError:
        if request.status_code == 404:
            raise requests.HTTPError('Tile URL resulted in a 404 error. '
                                     'Double-check your tile url:\n{}'.format(tile_url))
        elif request.status_code == 104:
            if max_retries > 0:
                os.wait(wait)
                max_retries -= 1
                request = _retryer(tile_url, wait, max_retries)
            else:
                raise requests.HTTPError('Connection reset by peer too many times.')
    return request

def howmany(w, s, e, n, zoom, verbose=True, ll=False):
    '''
    Number of tiles required for a given bounding box and a zoom level
    ...

    Arguments
    ---------
    w       : float
              West edge longitude
    s       : float
              South edge latitude
    e       : float
              East edge longitude
    n       : float
              Noth edge latitude
    zoom    : int
              Level of detail
    verbose : Boolean
              [Optional. Default=True] If True, print short message with
              number of tiles and zoom.
    ll      : Boolean
              [Optional. Default: False] If True, `w`, `s`, `e`, `n` are
              assumed to be lon/lat as opposed to Spherical Mercator.
    '''
    if not ll:
        # Convert w, s, e, n into lon/lat
        w, s = _sm2ll(w, s)
        e, n = _sm2ll(e, n)
    if zoom == 'auto':
        zoom = _calculate_zoom(w, s, e, n)
    tiles = len(list(mt.tiles(w, s, e, n, [zoom])))
    if verbose:
        print("Using zoom level %i, this will download %i tiles" % (zoom,
              tiles))
    return tiles

def bb2wdw(bb, rtr):
    '''
    Convert XY bounding box into the window of the tile raster
    ...

    Arguments
    ---------
    bb      : tuple
              (left, bottom, right, top) in the CRS of `rtr`
    rtr     : RasterReader
              Open rasterio raster from which the window will be extracted

    Returns
    -------
    window  : tuple
              ((row_start, row_stop), (col_start, col_stop))
    '''
    rbb = rtr.bounds
    xi = np.linspace(rbb.left, rbb.right, rtr.shape[1])
    yi = np.linspace(rbb.bottom, rbb.top, rtr.shape[0])

    window = ((rtr.shape[0] - yi.searchsorted(bb[3]),
              rtr.shape[0] - yi.searchsorted(bb[1])),
              (xi.searchsorted(bb[0]),
               xi.searchsorted(bb[2]))
              )
    return window

def _sm2ll(x, y):
    '''
    Transform Spherical Mercator coordinates point into lon/lat

    NOTE: Translated from the JS implementation in
    http://dotnetfollower.com/wordpress/2011/07/javascript-how-to-convert-mercator-sphere-coordinates-to-latitude-and-longitude/
    ...

    Arguments
    ---------
    x       : float
              Easting
    y       : float
              Northing

    Returns
    -------
    ll      : tuple
              lon/lat coordinates
    '''
    rMajor = 6378137.  # Equatorial Radius, QGS84
    shift = np.pi * rMajor
    lon = x / shift * 180.
    lat = y / shift * 180.
    lat = 180. / np.pi * (2. * np.arctan(np.exp(lat * np.pi / 180.)) - np.pi / 2.)
    return lon, lat

def _calculate_zoom(w, s, e, n):
    """Automatically choose a zoom level given a desired number of tiles.

    .. note:: all values are interpreted as latitude / longitutde.

    Parameters
    ----------
    w : float
        The western bbox edge.
    s : float
        The southern bbox edge.
    e : float
        The eastern bbox edge.
    n : float
        The northern bbox edge.

    Returns
    -------
    zoom : int
        The zoom level to use in order to download this number of tiles.
    """
    # Calculate bounds of the bbox
    lon_range = np.sort([e, w])[::-1]
    lat_range = np.sort([s, n])[::-1]

    lon_length = np.subtract(*lon_range)
    lat_length = np.subtract(*lat_range)

    # Calculate the zoom
    zoom_lon = np.ceil(np.log2(360 * 2. / lon_length))
    zoom_lat = np.ceil(np.log2(360 * 2. / lat_length))
    zoom = np.max([zoom_lon, zoom_lat])
    return int(zoom)

def _merge_tiles(tiles, arrays):
    """
    Merge a set of tiles into a single array.

    Parameters
    ---------
    tiles  : list of mercantile.Tile objects
             The tiles to merge.
    arrays : list of numpy arrays
             The corresponding arrays (image pixels) of the tiles. This list
             has the same length and order as the `tiles` argument.

    Returns
    -------
    img : np.ndarray
        Merged arrays.
    extent : tuple
         Bounding box [west, south, east, north] of the returned image
         in long/lat.
    """
    # create (n_tiles x 2) array with column for x and y coordinates
    tile_xys = np.array([(t.x, t.y) for t in tiles])

    # get indices starting at zero
    indices = tile_xys - tile_xys.min(axis=0)

    # the shape of individual tile images
    h, w, d = arrays[0].shape

    # number of rows and columns in the merged tile
    n_x, n_y = (indices+1).max(axis=0)

    # empty merged tiles array to be filled in
    img = np.zeros((h * n_y, w * n_x, d), dtype=np.uint8)

    for ind, arr in zip(indices, arrays):
        x, y = ind
        img[y*h:(y+1)*h, x*w:(x+1)*w, :] = arr

    bounds = np.array([mt.bounds(t) for t in tiles])
    west, south, east, north = (
        min(bounds[:, 0]), min(bounds[:, 1]),
        max(bounds[:, 2]), max(bounds[:, 3]))

    return img, (west, south, east, north)


'''
this scraping script is meant for POC using API for development
'''

import requests
import urllib.request
import json
from bs4 import BeautifulSoup
from threading import Thread
from .indeed_refactor import Indeed

def scrape( all_data , url, location, proxies):
    try:
        response = requests.get(url, headers={'User-Agent': 'Mozilla/5.0'} , proxies=proxies, timeout= 7)
        soup = BeautifulSoup(response.text, 'html.parser')

        #content_container= soup.find_all('a', {'class': ["result-card__full-card-link"]})
        content_container= soup.find_all('li', {'class': [  "result-card job-result-card result-card--with-hover-state"]})
    except requests.exceptions.ProxyError:
        return
    except requests.exceptions.ConnectTimeout:
        return
    except requests.exceptions.ConnectionError:
        return
    for x, i in enumerate(content_container):
        try:
            title=i.a.span.text
            href=i.a['href']
            company=i.h4.a.text
            time=i.time.text
            all_data[href] =[title, company, time, href ]
        except AttributeError:
            pass
    return all_data

def start(role , location):
    all_data={}
    allproxies=Indeed().getproxies()
    url= "https://www.linkedin.com/jobs/search?keywords="+ role + "&location=" + location+"&trk=public_jobs_jobs-search-bar_search-submit&redirect=false&position=1&pageNum="
    threads= [ Thread(target=scrape, args=( all_data,  url +str(n) , role , Indeed().get_session(allproxies)  ),daemon=True) for n in range(1, 20)]
    for t in threads:
        t.start()
    for t in threads:
        t.join()
    return all_data

if __name__ == "__main__":
    ret= start('python' , 'new york')


import tkinter as tk

import src.frames as frm
import src.preload as pl

class Application(tk.Tk):
    def __init__(self):
        super().__init__()

        self.title("Toovico")
        self.minsize(220, 200)
        pl.load_styles(self)

        self.ui = frm.MainUI(self)
        self.ui.notebook.add(frm.ConvertVideoTab(self), text="Convert video")
        self.ui.notebook.add(frm.ImageToVideoTab(self), text="Create video")

if __name__ == '__main__':
    app = Application()
    app.mainloop()


def mergeSortedLists(listA, listB):
    newList = list()
    a = 0
    b = 0

    while a < len(listA) and b < len(listB):
        if listA[a] < listB[b]:
            newList.append(listA[a])
            a += 1
        else:
            newList.append(listB[b])
            b += 1

    while a < len(listA):
        newList.append(listA[a])
        a += 1
    while b < len(listB):
        newList.append(listB[b])
        b += 1

    return newList
testlistA = [2, 8, 15, 23, 37]
testlistB = [4, 6, 15, 20]

print('List A ={}'.format(testlistA) + '\nList B ={}'.format(testlistB))
print('After merge sorted lists:{}'.format(mergeSortedLists(testlistA, testlistB)))


class Node:
    def __init__(self):
        self.children = []
        self.degree = 0

class Solution:
    def sequenceReconstruction(self, org: List[int], seqs: List[List[int]]) -> bool:
        # get all source
        sources = set()
        for seq in seqs:
            for s in seq:
                sources.add(s)

        # build
        graph = collections.defaultdict(Node)
        for seq in seqs:
            parent = seq[0]
            if parent not in graph:
                graph[parent] = Node()
            for child in seq[1:]:
                graph[parent].children.append(child)
                graph[child].degree += 1
                parent = child

        # get degree=0
        queue = collections.deque()
        for key, node in graph.items():
            if node.degree == 0:
                queue.appendleft(key)

        # bfs
        index = 0
        result = []
        while queue:
            if len(queue) > 1:
                return False
            node = queue.pop()

            if index >= len(org):
                return False
            if node != org[index]:
                return False

            index += 1
            result.append(node)
            for child in graph[node].children:
                graph[child].degree -= 1
                if graph[child].degree == 0:
                    queue.appendleft(child)

        return len(result) == len(sources) and result == org


"""
This file is the main method which tests if this code compiling well
"""
from Zoo.exceptions.exceptions import ConstructionNotExisting, TooLittleSpace
from manager.building_manager import BuildingManager
from manager.set_manager import SetManager
from model.bank import Bank
from model.library import Library
from model.school import School
from model.zoo import Zoo

if __name__ == "__main__":
    buildings = []
    manager = BuildingManager(buildings)
    manager.add_building(Zoo(49.4, 10, "Hoshin", "Tokyo", 2004, True), message="building add succesfully")
    manager.add_building(Zoo(), message="building add succesfully")
    manager.add_building(Library(5, 2000, "Stepana bandery", 1980, False), message="building add succesfully")
    manager.add_building(Library(), message="building add succesfully")
    manager.add_building(School(500, 30, "LPML", 2000, True), message="building add succesfully")
    manager.add_building(School(), message="building add succesfully")
    manager.add_building(Bank(5, "9:00-18:00", 100, False, 2005), message="building add succesfully")
    manager.add_building(Bank(), message="building add succesfully")
    buildings_build_after_1999 = manager.find_buildings_build_after(1999)
    residential_buildings = manager.find_residential_buildings()

    for building in residential_buildings:
        print(building)
    print("\n")
    for building in buildings_build_after_1999:
        print(building)
    print("\n")
    print(list(manager.pairs_of_building_and_construction_price()))
    print(Zoo(49.4, 10, "Hoshin", "Tokyo", 2004, True).all_values_with_type(int))
    set_manager1 = SetManager(manager)
    ism = iter(set_manager1)
    for i in ism:
        print(ism.__next__())
    print(ism)
    print(manager.is_buildings_build_in(2000))
    print(manager.is_any_residential_building())
    exception_zoo = Zoo(0, 0)
    try:
        exception_zoo.calculate_construction_price()
    except ConstructionNotExisting as exc:
        print(exc)
    try:
        exception_zoo.increase_capacity(100)
    except TooLittleSpace as exc:
        print(exc)


from heapq import heappush, heappop
import numpy as np
class Solution:
    def kClosest(self, points: List[List[int]], k: int) -> List[List[int]]:
        n=len(points)
        func=[]
        i=0
        heapq.heapify(func)
        for point in points:
            x=np.square(point[0])+np.square(point[1])
            func.append([x, point])
        func.sort()
        points.clear()
        while(k):
            x,y=heappop(func)
            points.append(y)
            k-=1
        return points

#         while(k!=0):

#             k-=1


#!/usr/bin/env python

from setuptools import setup, find_packages

requirements = [
    "kaggle==1.5.12",
    "numpy==1.23.3",
    "opencv-python-headless==4.6.0.66",
    "wandb==0.13.7",
    "pbjson==1.15",
    "hydra-core==1.2",
    "omegaconf==2.2.3",
    "kornia==0.6.8",
    "pandas==1.5.0",
    "scipy==1.9.1",
    "matplotlib==3.6.0",
]

test_requirements = []

dependency_links = [
]

setup(
    author="Massimiliano Biancucci",
    author_email="Binacucci95@Gmail.com",
    python_requires=">=3.7",
    classifiers=[
        "Development Status :: 3 - Alpha",
        "Intended Audience :: Developers",
        "Natural Language :: English",
        "Programming Language :: Python :: 3.8",
    ],
    description="Training, inference and test pipeline package for a controled masked adversarial generator",
    install_requires=requirements,
    long_description="",
    include_package_data=True,
    keywords="COIGAN",
    name="COIGAN",
    packages=find_packages(
        include=[
            "evaluation",
            "evaluation.*",
            "modules",
            "modules.*",
            "training",
            "training.*",
            "inference",
            "inference.*",
            "shape_training",
            "shape_training.*",
            "utils",
            "utils.*",
        ]
    ),
    dependency_links=dependency_links,
    test_suite="tests",
    tests_require=test_requirements,
    url="https://github.com/MassimilianoBiancucci/COIGAN-controllable-object-inpainting",
    version="0.1.0",
    zip_safe=False,
)