from github import Github
import yaml
import requests
import sys
import json

# get github token from command line
token = sys.argv[1]
g = Github(token)

# load org yaml
r = requests.get("https://raw.githubusercontent.com/github/government.github.com/gh-pages/_data/governments.yml")

data = None
try:
    data = yaml.safe_load(r.content)
except yaml.YAMLError as exc:
    print(exc)

# get organizations in Sweden
orgs = data["Sweden"]

# List organizations and their repos
for orgname in orgs:
    org = g.get_organization(orgname)
    for repo in org.get_repos(type="public"):
        if not repo.fork:
            j = {"orgname": org.name, "repo": repo.name, "repo_desc": repo.description, "repo_url": repo.html_url}
            print(json.dumps(j, indent=4))


import collections
import pprint
import time

from gensim.models.doc2vec import Doc2Vec, TaggedDocument
from gensim.models import KeyedVectors
from joblib import load
from nltk import word_tokenize, PorterStemmer
from nltk.corpus import stopwords

from algorithms.algorithm_utils import load_pickle, find_sub_matrix, recommend_restaurant
from algorithms.rating_based_algorithms import rating_recommend
from algorithms.text_based_algorithms import text_recommend
from users.models import UserReview, User

"""
This Python file provides some text-based algorithms: word2vec, Doc2vec and ....
Only support cosine similarity
"""

def test():
    filename = '../GoogleNews-vectors-negative300.bin'
    model = KeyedVectors.load_word2vec_format(filename, binary=True)
    # result = model.most_similar(positive=['woman', 'king'], negative=['man'], topn=1)
    print(model.n_similarity('I love you'.lower().split(), 'fuck you'.lower().split()))
    print('finished...')
    start = time.time()
    print(model.n_similarity('icecream I like apple apple apple'.lower().split(), 'hello you'.lower().split()))
    end = time.time()
    print(end - start)

def load_word2vec_model(filename):
    binary = False
    pre_path = 'algorithms/'
    if filename == 'GoogleNews-vectors-negative300.bin':
        binary = True
    return KeyedVectors.load_word2vec_format(pre_path + filename, binary=binary)

def word2vec_recommend(username, model):
    # find submatrix first
    user_restaurant_list = list()  # the restaurants the user went before.
    user_dict = dict()  # dict: user[business_id] = text_review
    user_text_list = list()  # the users text reviews list orders by user_dict
    user_review_list = UserReview.objects.filter(user=User.objects.get(username=username))

    for one_review in user_review_list:
        user_dict[one_review.business_id] = one_review.text
        user_restaurant_list.append(one_review.business_id)
        user_text_list.append(one_review.text)

    # now we have the user's text vector and we can calculate the similarity
    """
       first find the unrecorded restaurant
       find similar users from the dataset
       1. find the users list which the new user and customers have [threshold] common rating.
       this function is a recursive function: first the min of user_vector is 5,
       so, the threshold is 80% : 4.
    """

    # load text_matrix
    text_matrix = load_pickle('algorithms/text_matrix.pkl')
    threshold = len(user_restaurant_list) - 1  # means # of the common rating
    sub_matrix = find_sub_matrix(user_dict, text_matrix, threshold)
    while len(sub_matrix.keys()) < 30:
        threshold = threshold - 1
        sub_matrix = find_sub_matrix(user_dict, text_matrix, threshold)
    pprint.pprint(len(sub_matrix))

    # now we got the sub matrix and we can process
    similarity = dict()
    for one_similar_user in sub_matrix.keys():
        customer_total_text = ""
        user_total_text = ""
        for one_restaurant in user_restaurant_list:
            if one_restaurant in sub_matrix[one_similar_user]:
                customer_total_text = customer_total_text + ' ' + sub_matrix[one_similar_user][one_restaurant]
                user_total_text = user_total_text + ' ' + user_dict[one_restaurant]
        # now we have two list: user_total_text and customer_total. we can calculate the similarity
        # in word2vec n_similarity.
        sim_score = 0
        # tokenize the text
        customer_tokens = tokenizer(customer_total_text, model)
        user_tokens = tokenizer(user_total_text, model)

        similarity[one_similar_user] = model.n_similarity(customer_tokens, user_tokens)
    # get top 15 similar users
    top_similar_users = collections.Counter(similarity).most_common(15)
    return top_similar_users, recommend_restaurant(similarity, user_dict, username)

def tokenizer(input_text, wv_model):
    # get tokens
    tokens = word_tokenize(input_text)
    clean_tokens = list()
    for token in tokens:
        if token.lower() in wv_model.wv.vocab:
            clean_tokens.append(token.lower())
    return clean_tokens

def linear_regression_model(username, similarity_method='cosine', method='bag_of_words'):
    similar_user_text_based, similar_restaurant_text_based = text_recommend(username=username, method=method,
                                                                            similarity_method=similarity_method)
    similar_user_rating_based, similar_restaurant_rating_based = rating_recommend(username,
                                                                                similarity_method=similarity_method)
    # load linear regression model
    liner_regression_model = load('algorithms/trained_model.joblib')
    # prepare the X values, first find all the restaurant list.
    restaurant_list = list()
    for one in range(len(similar_restaurant_rating_based)):
        if similar_restaurant_rating_based[one][0] not in restaurant_list:
            restaurant_list.append(similar_restaurant_rating_based[one][0])
        if similar_restaurant_text_based[one][0] not in restaurant_list:
            restaurant_list.append(similar_restaurant_text_based[one][0])
    result = dict()
    X = list()
    for one_rest in restaurant_list:
        # get rating score
        r_score = get_score(similar_restaurant_rating_based, one_rest)
        t_score = get_score(similar_restaurant_text_based, one_rest)
        X.append([r_score, t_score])
    # use model to predict
    Y = liner_regression_model.predict(X)
    count = 0
    for one in restaurant_list:
        result[one] = Y[count][0]
        count = count + 1
    return similar_user_rating_based, similar_user_text_based, collections.Counter(result).most_common(5)

def get_score(similar_restaurants, restaurant_id):
    for one in similar_restaurants:
        if restaurant_id == one[0]:
            return one[1]
    return 0

def doc2vec_recommend(username, model):

    return word2vec_recommend(username, model.wv)
# if __name__ == '__main__':
#     print('loading...')
#     model = load_word2vec_model('glove.6B.50d.txt.word2vec')
#     print('Finish loading')
#     start = time.time()
#     simi_user, simi_rec = word2vec_recommend('123', model)
#     end = time.time()
#     print(end - start)
#     pprint.pprint(simi_rec)
#     pprint.pprint(simi_user)
#     exit()


from app import app
from datetime import datetime
from flask_login import current_user
from app.stratified import high_perspective
from app.survey import survey
from bson.objectid import ObjectId

query_classifier_toxic = \
    {"$and":[
        {"toxicity."+app.config['VERSION']+".score":1},
        {"toxicity."+app.config['VERSION']+".orig.persp_raw.detectedLanguages":["en"]},
        {"toxicity."+app.config['VERSION']+".en":{"$gt": .001}}
        ]
    }

query_has_predicted_toxic_comment = \
    {"toxicity."+app.config['VERSION']+".has_predicted_toxic_comment":True
    }

query_predicted_issues_all = \
    {"$or":[
        query_has_predicted_toxic_comment,
        query_classifier_toxic,
        ]
    }

query_predicted_prs_all = \
    {"$and":[
        {"$or":[
            query_has_predicted_toxic_comment,
            query_classifier_toxic,
            ]
        },
        {"is_pr":True}]
    }

query_predicted_issues_w_comments = \
    {"$and":[
            {"$or":[
                query_has_predicted_toxic_comment,
                query_classifier_toxic,
                ]
            },
            {"num_comments":{"$gt":0}}
        ]
    }

query_predicted_prs_w_comments = \
    {"$and":[
            {"$or":[
                query_has_predicted_toxic_comment,
                query_classifier_toxic,
                ]
            },
            {"num_comments":{"$gt":0}},
            {"is_pr":True}]
    }

query_predicted_prs_w_review_comments = \
    {"$and":[
            {"has_toxic_review_comment":True},
            {"num_comments":{"$gt":0}},
            {"is_pr":True}]
    }

query_ck_annotations = \
    {"$or":[
        {"toxicity.manual.score":1},
        {"toxicity.manual.score":0},
        ]
    }

query_annotations = {"has_labeled_comment":True}

query_annotations_toxic = \
    {"$or":[
        {"has_labeled_toxic_comment":True},
        {"is_labeled_toxic":True},
        ]
    }

query_confirmed_toxic = \
    {"$or":[
        {"toxicity.manual.score":1},
        {"toxicity.manual.score":0},
        {"has_labeled_comment":True},
        ]
    }

query_unlabeled = \
        {'$and': [{'is_labeled': {'$not':{'$eq': True}}}, {'has_labeled_comment': {'$not': {'$eq': True}}},{"updated_at":{"$regex": "^2020-0[345]"}}]}
query_labeled_toxic = \
        {"$or" : [{"is_labeled_toxic":True}, {"has_labeled_toxic_comment": True}]}

def query_individual_annotations(username):
    return {"$and":[
        {"$or":[
            {"has_labeled_comment":True},
            {"is_labeled": True}
            ]
        },
        {"toxicity.manual_labels.user":username},
        ]
    }

def query_tolabel(username):
    return {"$and":[
        {"toxicity.manual_labels.user":{"$not": {"$regex": "^"+username+"$"}}},
        {"toxicity.manual_labeled_comments.user":{"$not": {"$regex": "^"+username+"$"}}},
        {"toxicity.todo":1}
        ]
    }

def query_closed(username):
    return {"$and":[
        {"state" : "closed" },
        {"merged" : False },
        {"$or":
            [ {"comments": {"$gte": 10} },
              {"review_comments": {"$gte": 10} },
            ]
        },
        {"changed_files": {"$lte": 5} },
        {"created_at": {"$gte": "2020-01-01"} }
        ]
    }

def query_survey(username):
    return {"$and":[
        {"_id" : { "$in" : [ObjectId(_id) for _id in survey] } },
        {"toxicity."+app.config['VERSION']+".orig.persp_raw.detectedLanguages":["en"]},
        {"toxicity."+app.config['VERSION']+".en":{"$gt": .001}},
        {"toxicity.manual_labels.user":{"$not": {"$regex": "^"+username+"$"}}},
        {"toxicity.manual_labeled_comments.user":{"$not": {"$regex": "^"+username+"$"}}},
        ]
    }

query_stratified = \
    {"$and":[
        {"_id" : { "$in" : [ObjectId(_id) for _id in high_perspective] } },
        {"toxicity."+app.config['VERSION']+".orig.persp_raw.detectedLanguages":["en"]},
        {"toxicity."+app.config['VERSION']+".en":{"$gt": .001}}
        ]
    }

def qand(a, b):
    return {"$and": [a,b]}
def qunlabeled(a):
    return qand(a, query_unlabeled)
def qlabeled_toxic(a):
    return qand(a, query_labeled_toxic)

query_locked = {"is_locked_too_heated":True}
query_deleted= {"is_deleted":True}
query_toxiccomment= {"has_comment_classified_toxic":True}
query_coc= {"has_comment_mention_code_of_conduct":True}


from django.contrib import auth
from django.http import HttpResponseRedirect
from rest_framework.views import APIView

class LogOutView(APIView):
    def get(self, request):
        auth.logout(request)
        response = HttpResponseRedirect("/")
        cookie_keys = ["bk_ticket", "bk_token", "bk_uid"]
        host = "".join(request.headers["Host"].split(":")[:1])
        domain = f'.{".".join(host.split(".")[1:])}'
        for _domain in [host, domain]:
            for key in cookie_keys:
                response.delete_cookie(key, domain=_domain)
        return response


from __future__ import annotations
from typing import (
    Iterable,
    Any,
    Iterator,
    Sequence,
    Callable,
    Optional,
    NamedTuple,
)
from typing_extensions import Self, TypeAlias
import copy
import enum
import dataclasses

from ezdxf.math import BoundingBox2d, Matrix44, Vec2, UVec
from ezdxf.npshapes import NumpyPath2d, NumpyPoints2d, EmptyShapeError
from ezdxf.tools import take2

from .backend import BackendInterface
from .config import Configuration
from .properties import BackendProperties
from .type_hints import Color

class RecordType(enum.Enum):
    """Enum, determines the data record type.

    Attributes:
        POINTS:
        SOLID_LINES:
        PATH:
        FILLED_PATHS:
    """

    POINTS = enum.auto()  # n=1 point; n=2 line; n>2 filled polygon
    SOLID_LINES = enum.auto()
    PATH = enum.auto()
    FILLED_PATHS = enum.auto()

@dataclasses.dataclass
class DataRecord:
    type: RecordType
    property_hash: int
    handle: str  # top-level entity handle
    data: Any

    def bbox(self) -> BoundingBox2d:
        bbox = BoundingBox2d()
        try:
            if self.type == RecordType.FILLED_PATHS:
                for path in self.data:
                    bbox.extend(path.extents())
            else:
                bbox.extend(self.data.extents())
        except EmptyShapeError:
            pass
        return bbox

class Recorder(BackendInterface):
    """Records the output of the Frontend class."""

    def __init__(self) -> None:
        self.config = Configuration()
        self.background: Color = "#000000"
        self.records: list[DataRecord] = []
        self.properties: dict[int, BackendProperties] = dict()

    def player(self) -> Player:
        """Returns a :class:`Player` instance with the original recordings! Make a copy
        of this player to protect the original recordings from being modified::

            safe_player = recorder.player().copy()

        """
        player = Player()
        player.config = self.config
        player.background = self.background
        player.records = self.records
        player.properties = self.properties
        player.has_shared_recordings = True
        return player

    def configure(self, config: Configuration) -> None:
        self.config = config

    def set_background(self, color: Color) -> None:
        self.background = color

    def store(
        self, type_: RecordType, properties: BackendProperties, data: Any
    ) -> None:
        # exclude top-level entity handle to reduce the variance:
        # color, lineweight, layer, pen
        prop_hash = hash(properties[:4])
        self.records.append(
            DataRecord(
                type=type_, property_hash=prop_hash, handle=properties.handle, data=data
            )
        )
        self.properties[prop_hash] = properties

    def draw_point(self, pos: Vec2, properties: BackendProperties) -> None:
        self.store(RecordType.POINTS, properties, NumpyPoints2d((pos,)))

    def draw_line(self, start: Vec2, end: Vec2, properties: BackendProperties) -> None:
        self.store(RecordType.POINTS, properties, NumpyPoints2d((start, end)))

    def draw_solid_lines(
        self, lines: Iterable[tuple[Vec2, Vec2]], properties: BackendProperties
    ) -> None:
        def flatten() -> Iterator[Vec2]:
            for s, e in lines:
                yield s
                yield e

        self.store(RecordType.SOLID_LINES, properties, NumpyPoints2d(flatten()))

    def draw_path(self, path: NumpyPath2d, properties: BackendProperties) -> None:
        assert isinstance(path, NumpyPath2d)
        self.store(RecordType.PATH, properties, path)

    def draw_filled_polygon(
        self, points: NumpyPoints2d, properties: BackendProperties
    ) -> None:
        assert isinstance(points, NumpyPoints2d)
        self.store(RecordType.POINTS, properties, points)

    def draw_filled_paths(
        self, paths: Iterable[NumpyPath2d], properties: BackendProperties
    ) -> None:
        paths = tuple(paths)
        if len(paths) == 0:
            return

        assert isinstance(paths[0], NumpyPath2d)
        self.store(RecordType.FILLED_PATHS, properties, paths)

    def enter_entity(self, entity, properties) -> None:
        pass

    def exit_entity(self, entity) -> None:
        pass

    def clear(self) -> None:
        raise NotImplementedError()

    def finalize(self) -> None:
        pass

class Override(NamedTuple):
    """Represents the override state for a data record.

    Attributes:
        properties: original or modified :class:`BackendProperties`
        is_visible: override visibility e.g. switch layers on/off

    """

    properties: BackendProperties
    is_visible: bool = True

OverrideFunc: TypeAlias = Callable[[BackendProperties], Override]

class Player:
    """Plays the recordings of the :class:`Recorder` backend on another backend."""

    def __init__(self) -> None:
        self.config = Configuration()
        self.background: Color = "#000000"
        self.records: list[DataRecord] = []
        self.properties: dict[int, BackendProperties] = dict()
        self._bbox = BoundingBox2d()
        self.has_shared_recordings: bool = False

    def __copy__(self) -> Self:
        """Returns a copy of the player with non-shared recordings."""
        player = self.__class__()
        # config is a frozen dataclass:
        player.config = self.config
        player.background = self.background
        # recordings are mutable - transformed inplace:
        player.records = copy.deepcopy(self.records)
        # the properties dict may grow, but entries will never be removed:
        player.properties = self.properties
        player.has_shared_recordings = False
        return player

    copy = __copy__

    def recordings(self) -> Iterator[tuple[RecordType, BackendProperties, Any]]:
        """Yields all recordings as `(RecordType, BackendProperties, Data)` tuples.

        The content of the `Data` field is determined by the enum :class:`RecordType`:

        - :attr:`RecordType.POINTS` returns a :class:`NumpyPoints2d` instance,
          len() == 1 is a point, len() == 2 is a line, len() > 2 is a filled polygon
        - :attr:`RecordType.SOLID_LINES` returns a :class:`NumpyPoints2d` instance
          where each pair (n, n+1) represents the start- and end point of a line
        - :attr:`RecordType.PATH`: returns a :class:`NumpyPath2d` instance that
          represents a linear 2D path
        - :attr:`RecordType.FILLED_PATHS` returns a tuple (exterior_paths, holes),
          where exterior_paths and holes are tuples of :class:`NumpyPath2d`.

        """
        props = self.properties
        for record in self.records:
            properties = BackendProperties(
                *props[record.property_hash][:4], record.handle
            )
            yield record.type, properties, record.data

    def replay(
        self, backend: BackendInterface, override: Optional[OverrideFunc] = None
    ) -> None:
        """Replay the recording on another backend that implements the
        :class:`BackendInterface`. The optional `override` function can be used to
        override the properties and state of data records, it gets the :class:`BackendProperties`
        as input and must return an :class:`Override` instance.
        """

        backend.configure(self.config)
        backend.set_background(self.background)
        for record_type, properties, data in self.recordings():
            if override:
                state = override(properties)
                if not state.is_visible:
                    continue
                properties = state.properties
            if record_type == RecordType.POINTS:
                if len(data) == 0:
                    continue
                if len(data) > 2:
                    backend.draw_filled_polygon(data, properties)
                    continue
                vertices = data.vertices()
                if len(vertices) == 1:
                    backend.draw_point(vertices[0], properties)
                else:
                    backend.draw_line(vertices[0], vertices[1], properties)
            elif record_type == RecordType.SOLID_LINES:
                backend.draw_solid_lines(take2(data.vertices()), properties)
            elif record_type == RecordType.PATH:
                backend.draw_path(data, properties)
            elif record_type == RecordType.FILLED_PATHS:
                backend.draw_filled_paths(data, properties)
        backend.finalize()

    def transform(self, m: Matrix44) -> None:
        """Transforms the recordings inplace by a transformation matrix `m` of type
        :class:`~ezdxf.math.Matrix44`.
        """
        for record in self.records:
            if record.type == RecordType.FILLED_PATHS:
                for p in record.data:
                    p.transform_inplace(m)
            else:
                record.data.transform_inplace(m)

        if self._bbox.has_data:
            # works for 90-, 180- and 270-degree rotation
            self._bbox = BoundingBox2d(m.fast_2d_transform(self._bbox.rect_vertices()))

    def bbox(self) -> BoundingBox2d:
        """Returns the bounding box of all records as :class:`~ezdxf.math.BoundingBox2d`."""
        if not self._bbox.has_data:
            self.update_bbox()
        return self._bbox

    def update_bbox(self) -> None:
        bbox = BoundingBox2d()
        for record in self.records:
            bbox.extend(record.bbox())
        self._bbox = bbox

    def crop_rect(self, p1: UVec, p2: UVec, distance: float) -> None:
        """Crop recorded shapes inplace by a rectangle defined by two points.

        The argument `distance` defines the approximation precision for paths which have
        to be approximated as polylines for cropping but only paths which are really get
        cropped are approximated, paths that are fully inside the crop box will not be
        approximated.

        Args:
            p1: first corner of the clipping rectangle
            p2: second corner of the clipping rectangle
            distance: maximum distance from the center of the curve to the
                center of the line segment between two approximation points to
                determine if a segment should be subdivided.

        """
        crop_rect = BoundingBox2d([Vec2(p1), Vec2(p2)])
        self.records = crop_records_rect(self.records, crop_rect, distance)
        self._bbox = BoundingBox2d()  # determine new bounding box on demand

def crop_records_rect(
    records: list[DataRecord], crop_rect: BoundingBox2d, distance: float
) -> list[DataRecord]:
    """Crop recorded shapes inplace by a rectangle."""
    from .clipper import ClippingRect

    def sort_paths(np_paths: Sequence[NumpyPath2d]):
        _inside: list[NumpyPath2d] = []
        _crop: list[NumpyPath2d] = []

        for np_path in np_paths:
            bbox = BoundingBox2d(np_path.extents())
            if not crop_rect.has_intersection(bbox):
                # path is complete outside the cropping rectangle
                pass
            elif crop_rect.inside(bbox.extmin) and crop_rect.inside(bbox.extmax):
                # path is complete inside the cropping rectangle
                _inside.append(np_path)
            else:
                _crop.append(np_path)

        return _crop, _inside

    def crop_paths(
        np_paths: Sequence[NumpyPath2d],
    ) -> list[NumpyPath2d]:
        return list(clipper.clip_filled_paths(np_paths, distance))

    # an undefined crop box crops nothing:
    if not crop_rect.has_data:
        return records
    cropped_records: list[DataRecord] = []
    size = crop_rect.size
    # a crop box size of zero in any dimension crops everything:
    if size.x < 1e-12 or size.y < 1e-12:
        return cropped_records

    clipper = ClippingRect()
    clipper.push(NumpyPath2d.from_vertices(crop_rect.rect_vertices()), None)
    for record in records:
        record_box = record.bbox()
        if not crop_rect.has_intersection(record_box):
            # record is complete outside the cropping rectangle
            continue
        if crop_rect.inside(record_box.extmin) and crop_rect.inside(record_box.extmax):
            # record is complete inside the cropping rectangle
            cropped_records.append(record)
            continue

        if record.type == RecordType.FILLED_PATHS:
            paths_to_crop, inside = sort_paths(record.data)
            cropped_paths = crop_paths(paths_to_crop) + inside
            if cropped_paths:
                record.data = tuple(cropped_paths)
                cropped_records.append(record)
        elif record.type == RecordType.PATH:
            # could be split into multiple parts
            for p in clipper.clip_paths([record.data], distance):  # type: ignore
                cropped_records.append(
                    DataRecord(
                        record.type,
                        record.property_hash,
                        record.handle,
                        p,
                    )
                )
        elif record.type == RecordType.POINTS:
            count = len(record.data)
            if count == 1:
                pass
            elif count == 2:
                s, e = record.data.vertices()
                record.data = NumpyPoints2d(clipper.clip_line(s, e))
            else:  # filled polygon
                record.data = clipper.clip_polygon(record.data)
            cropped_records.append(record)
        elif record.type == RecordType.SOLID_LINES:
            points: list[Vec2] = []  # type: ignore
            for s, e in take2(record.data.vertices()):
                points.append(clipper.clip_line(s, e))  # type: ignore
            record.data = NumpyPoints2d(points)
            cropped_records.append(record)
        else:
            raise ValueError("invalid record type")
    return cropped_records


import numpy as np
import matplotlib.pyplot as plt
from bg_atlasapi import BrainGlobeAtlas
from skimage.filters import gaussian

from cordmap.postprocessing.prob_map import get_bins

atlas_name = "salk_cord_10um"
atlas = BrainGlobeAtlas(atlas_name)

def generate_histogram2d(points_file, pos, smoothing=100, axis="x"):

    points = np.load(points_file)
    if axis == "z":
        atlas_resolution = atlas.resolution[1:]
        points_slice = points[np.where(points[:, 0] == pos)]
        points_slice = points_slice[:, 1:]
        bins = get_bins(atlas.annotation.shape[1:], (1, 1))
        atlas_image = atlas.annotation[pos, :, :]
    elif axis == "y":
        atlas_resolution = atlas.resolution[::2]
        points_slice = points[np.where(points[:, 1] == pos)]
        points_slice = points_slice[:, ::2]
        bins = get_bins(atlas.annotation.shape[::2], (1, 1))
        atlas_image = atlas.annotation[:, pos, :]
    elif axis == "x":
        atlas_resolution = atlas.resolution[:-1]
        points_slice = points[np.where(points[:, 2] == pos)]
        points_slice = points_slice[:, :-1]
        bins = get_bins(atlas.annotation.shape[:-1], (1, 1))
        atlas_image = atlas.annotation[:, :, pos]
    probability_map, _ = np.histogramdd(points_slice, bins=bins, density=True)

    if smoothing is not None:
        smoothing = [int(round(smoothing / res)) for res in atlas_resolution]
        probability_map = gaussian(probability_map, sigma=smoothing)

    plt.figure()
    plt.imshow(probability_map)
    [plt.plot(p[1], p[0], "o", color="w") for p in points_slice]

    plt.figure()
    plt.imshow(atlas_image)
    [plt.plot(p[1], p[0], "o", color="w") for p in points_slice]
    plt.show()


# 3.1
import numpy as np
import cv2 as cv

# function for convolution with kernel
def Convolve_Kernel(img, kernel):
    rows,cols=img.shape
    row_k,col_k = kernel.shape
    new_img= np.zeros((rows,cols)).astype(np.uint8)
    center = int(row_k/2)
    for x in range(0,rows-row_k):
        for y in range(0,cols-col_k):
            sum = 0
            for i in range(0,row_k):
                for j in range(0,col_k):
                    sum = sum + kernel[i,j]*img[i+x,j+y]
            #print(x+center,y+center)
            new_img[x+center,y+center]= np.absolute(sum)
    return new_img

img = cv.imread('D:/DIP/Assignment_2/a2_2019701006/input_data/box.png',0)
kernel = np.array([[1,1,1],[1,0,-1],[-1,-1,-1]])
#Applying convolvution
new_img = Convolve_Kernel(img,kernel)
final = cv.hconcat([img,new_img])
cv.imshow('White Line',final)
cv.waitKey(0)


import sqlite3
conn = sqlite3.connect ('readersclub_website/tests/Emp.db')
cur = conn.cursor()
#cur.execute("CREATE TABLE Employee_Record(Emp_ID INTEGER PRIMARY KEY AUTOINCREMENT, Emp_Name TEXT, Age INTEGER);")
print("table created")

emp_name=input('Whats your name')
emp_age=int(input('Whats your age'))

cur.execute('INSERT INTO Employee_Record(Emp_Name,Age) VALUES(?,?);',[emp_name,emp_age])
conn.commit()
conn.close()


"""Unit test is expected to be here, while we use some usage cases instead."""

import logging
import os.path as osp
import shutil
import tempfile

from icrawler.builtin import BaiduImageCrawler, BingImageCrawler, GoogleImageCrawler, GreedyImageCrawler, UrlListCrawler

test_dir = tempfile.mkdtemp()

def test_google():
    img_dir = osp.join(test_dir, "google")
    google_crawler = GoogleImageCrawler(downloader_threads=2, storage={"root_dir": img_dir}, log_level=logging.INFO)
    search_filters = dict(size="large", color="orange", license="commercial,modify", date=(None, (2017, 11, 30)))
    google_crawler.crawl("cat", filters=search_filters, max_num=5)

def test_bing():
    img_dir = osp.join(test_dir, "bing")
    bing_crawler = BingImageCrawler(downloader_threads=2, storage={"root_dir": img_dir}, log_level=logging.INFO)
    search_filters = dict(type="photo", license="commercial", layout="wide", size="large", date="pastmonth")
    bing_crawler.crawl("cat", max_num=5, filters=search_filters)

def test_baidu():
    img_dir = osp.join(test_dir, "baidu")
    search_filters = dict(size="large", color="blue")
    baidu_crawler = BaiduImageCrawler(downloader_threads=2, storage={"root_dir": img_dir})
    baidu_crawler.crawl("cat", filters=search_filters, max_num=5)

def test_greedy():
    img_dir = osp.join(test_dir, "greedy")
    greedy_crawler = GreedyImageCrawler(parser_threads=2, storage={"root_dir": img_dir})
    greedy_crawler.crawl("http://www.bbc.com/news", max_num=5, min_size=(100, 100))

def test_urllist():
    img_dir = osp.join(test_dir, "urllist")
    urllist_crawler = UrlListCrawler(downloader_threads=2, storage={"root_dir": img_dir})
    filelist = osp.join(osp.dirname(osp.dirname(__file__)), "examples/filelist_demo.txt")
    urllist_crawler.crawl(filelist)


import argparse
import os
from glob import glob

from app import get_results
from configs import config

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--start_period", type=int, help="Start period")
    parser.add_argument("--end_period", type=int, help="End period")
    args = parser.parse_args()

    start_period = args.start_period
    end_period = args.end_period

    sales_paths = glob(pathname=os.path.join(config.SALES_DIR, "*.xls*"))
    expenses_paths = glob(pathname=os.path.join(config.EXPENSES_DIR, "*.csv"))

    get_results(
        sales_data=sales_paths,
        expenses_data=expenses_paths,
        start_period=start_period,
        end_period=end_period,
    )
    exit()


# -*- coding: utf-8 -*-
#!/usr/bin/env python
'''
By Dabi Ahn. andabi412@gmail.com.
https://www.github.com/andabi
'''

# import matplotlib.pyplot as plt
import librosa
# import librosa.display
import numpy as np

filename = 'dataset/ikala/Wavfile/21045_verse.wav'
sr = 16000
n_fft = 4096
len_hop = n_fft / 4

########################################################################################################################
# Wav to STFT
########################################################################################################################

# Read 2-channel wav files
data, sr = librosa.load(filename, sr=sr, mono=False)

# Split wav into music and vocal parts
music_wav, vocal_wav = data
mix_wav = librosa.to_mono(data)

# Write each wav
librosa.output.write_wav('music.wav', music_wav, sr)
librosa.output.write_wav('vocal.wav', vocal_wav, sr)
librosa.output.write_wav('mix.wav', mix_wav, sr)

# Transform to spectrogram
D_music = librosa.stft(music_wav, n_fft=n_fft, hop_length=len_hop)
D_vocal = librosa.stft(vocal_wav, n_fft=n_fft, hop_length=len_hop)
D_mix = librosa.stft(mix_wav, n_fft=n_fft, hop_length=len_hop)

# magnitude and phase
magnitude_music, phase_music = np.abs(D_music), np.angle(D_music)
magnitude_vocal, phase_vocal = np.abs(D_vocal), np.angle(D_vocal)
magnitude_mix, phase_mix = np.abs(D_mix), np.angle(D_mix)

########################################################################################################################
# STFT to wav
########################################################################################################################

# Transform back to wav
music_wav_recon = librosa.istft(D_music, hop_length=len_hop)
vocal_wav_recon = librosa.istft(D_vocal, hop_length=len_hop)
mix_wav_recon = librosa.istft(D_mix, hop_length=len_hop)

# Write each reconstructed wav
librosa.output.write_wav('music_recon.wav', music_wav_recon, sr)
librosa.output.write_wav('vocal_recon.wav', vocal_wav_recon, sr)
librosa.output.write_wav('mix_recon.wav', mix_wav_recon, sr)

########################################################################################################################
# Display
########################################################################################################################

# Plot waveforms
# plt.figure(1)
# plt.subplot(3, 1, 1)
# librosa.display.waveplot(mix_wav, sr=sr, color='r')
# plt.title('mix')
#
# plt.subplot(3, 1, 2)
# librosa.display.waveplot(music_wav, sr=sr)
# plt.title('music')
#
# plt.subplot(3, 1, 3)
# librosa.display.waveplot(vocal_wav, sr=sr)
# plt.title('vocal')
# plt.tight_layout()
#
# # Plot spectrogram
# plt.figure(2)
# plt.subplot(3, 1, 1)
# librosa.display.specshow(librosa.amplitude_to_db(D_music, ref=np.max), sr=sr, hop_length=len_hop, y_axis='log', x_axis='time')
# plt.title('music spectrogram')
# plt.colorbar(format='%+2.0f dB')
#
# plt.subplot(3, 1, 2)
# librosa.display.specshow(librosa.amplitude_to_db(D_vocal, ref=np.max), sr=sr, hop_length=len_hop, y_axis='log', x_axis='time')
# plt.title('vocal spectrogram')
# plt.colorbar(format='%+2.0f dB')
# plt.tight_layout()
#
# plt.subplot(3, 1, 3)
# librosa.display.specshow(librosa.amplitude_to_db(D_mix, ref=np.max), sr=sr, hop_length=len_hop, y_axis='log', x_axis='time')
# plt.title('mix spectrogram')
# plt.colorbar(format='%+2.0f dB')

# plt.show()


"""
Code adapted from poly_dust_detector.py
https://github.com/taller-de-robotica/1er-taller-pc/blob/main/paneles/poly_dust_detector.py
@author: crat2
"""

#LIBRARIES
import numpy as np
from matplotlib import pyplot as plt
from patchify import patchify, unpatchify
import cv2

from termcolor import cprint

import segmentation_models as sm
from keras.models import load_model

_MC = 'yellow'
_image_size = (1792, 1024)

class Unet_Model:

    def __init__(self, model_path) -> None:
        cprint("Loading model...", _MC)
        self.model = load_model(model_path, compile=False)

    #UNET_PREDICTION FUNCTION
    def unet_prediction(self, image):
        """
        Receives an RGB image and returns a 1D numpy array of ints where
        0 - Panel
        1 - Background
        2 - Dust
        """
        patch_size = 256
        unet_model = self.model

        #Patches preprocessing
        cprint("Creating patches...", _MC)
        small_patches = []
        patches_img = patchify(image, (patch_size, patch_size, 3), step=patch_size) ##perform patchify
        for i in range(patches_img.shape[0]):
            for j in range(patches_img.shape[1]):
                single_patch_img = patches_img[i,j,:,:]
                small_patches.append(single_patch_img)
        small_patches =  np.array(small_patches)

        #Unet4 SM preprocessing
        cprint("Preprocessing with resnet50...", _MC)
        BACKBONE = 'resnet50' #define the backbone
        preprocess_input = sm.get_preprocessing(BACKBONE)
        norm_img = preprocess_input(small_patches)
        norm_img =  np.array(norm_img)
        norm_img = np.squeeze(norm_img, 1)

        #Prediction
        cprint("Predicting...", _MC)
        y_pred_unet = unet_model.predict(norm_img) #make the prediction
        prediction_unet = np.argmax(y_pred_unet, axis=3)[:,:,:] #from prob to int

        #Reconstructed image
        cprint("Reconstructing image...", _MC)
        patched_prediction = np.reshape(prediction_unet, [patches_img.shape[0], patches_img.shape[1],
                                                        patches_img.shape[3], patches_img.shape[4]])
        reconstructed_image = unpatchify(patched_prediction, (image.shape[0], image.shape[1]))
        cprint("Finish..", _MC)
        return reconstructed_image

    #OUTPUT: Plotting original and GT and prediction
    def show_images(original_image, dust_image, show = False):
        """
        Displays the original image and image with detected regions.
        """
        cprint("Plotting image...", _MC)
        scale = 5
        new_size = (_image_size[0]//scale, _image_size[1]//scale)
        original = cv2.resize(original_image.astype(np.uint8), new_size)
        original = cv2.cvtColor(original, cv2.COLOR_RGB2BGR)

        reconstructed = cv2.resize(dust_image.astype(np.uint8), new_size) * 100
        reconstructed = cv2.merge((reconstructed,reconstructed,reconstructed))
        twins = np.concatenate((original, reconstructed), axis=1)

        if show:
            ## Funciona en Ubuntu, debe funcionar también en Windows
            cprint('Max value: ' + str(np.max(dust_image)), _MC)
            cprint("Press any key on the image to exit", _MC)
            cv2.namedWindow("Image vs Unet prediction", cv2.WINDOW_GUI_NORMAL | cv2.WINDOW_AUTOSIZE)
            cv2.imshow('Image vs Unet prediction', twins)
            cv2.waitKey(500)
            cv2.destroyAllWindows()

        return twins


"""
Pipeline for running, analyzing, saving, and loading PAX simulations.
"""

import numpy as np
import os
import pickle
from sklearn.model_selection import GridSearchCV
import pprint
from joblib import Parallel, delayed

from pax_deconvolve.deconvolution import deconvolvers
from pax_deconvolve.pax_simulations import simulate_pax

# Set global simulation parameters
PROCESSED_DATA_DIR = os.path.join(os.path.dirname(__file__), "simulated_results")
# Set default simulation parameters
DEFAULT_PARAMETERS = {
    "energy_loss": np.arange(-8, 10, 0.01),
    "iterations": 1e5,
    "simulations": 1000,
    "cv_fold": 3,
    "regularizer_widths": np.logspace(-3, -1, 10),
}
# good energy_loss for Ag 3d levels with Schlappa RIXS: np.arange(-8, 10, 0.005)
# good energy_loss for Fermi edge and doublet with < 0.4 eV separation: np.arange(-0.5, 0.5, 0.001)
# good regularizer_widths for Ag 3d: np.logspace(-3, -1, 10)
# good regularizer_widths for Fermi edge: np.logspace(-4, -2, 10)

def run(
    log10_num_electrons,
    rixs="schlappa",
    photoemission="ag",
    num_additional=25,
    **kwargs
):
    parameters = DEFAULT_PARAMETERS
    parameters.update(kwargs)
    print("Starting cv deconvolver")
    cv_deconvolver, pax_spectra = _run_cv(
        log10_num_electrons,
        rixs,
        photoemission,
        parameters["regularizer_widths"],
        parameters,
    )
    regularization_strength = cv_deconvolver.best_regularization_strength_
    print("Completed cv deconvolver")
    additional_deconvolutions = Parallel(n_jobs=-1)(
        delayed(_run_single_regularizer)(
            log10_num_electrons,
            rixs,
            photoemission,
            regularization_strength,
            parameters,
        )
        for _ in range(num_additional)
    )
    to_save = {
        "cv_deconvolver": cv_deconvolver,
        "additional_deconvolutions": additional_deconvolutions,
        "pax_spectra": pax_spectra,
    }
    file_name = _get_filename(log10_num_electrons, rixs, photoemission)
    with open(file_name, "wb") as f:
        pickle.dump(to_save, f)
    return to_save

def _run_cv(
    log10_num_electrons, rixs, photoemission, regularization_strengths, parameters
):
    impulse_response, pax_spectra, xray_xy = simulate_pax.simulate_from_presets(
        log10_num_electrons,
        rixs,
        photoemission,
        parameters["simulations"],
        parameters["energy_loss"],
    )
    deconvolver = deconvolvers.LRFisterGrid(
        impulse_response["x"],
        impulse_response["y"],
        pax_spectra["x"],
        parameters["regularizer_widths"],
        parameters["iterations"],
        xray_xy["y"],
        parameters["cv_fold"],
    )
    deconvolver.fit(np.array(pax_spectra["y"]))
    return deconvolver, pax_spectra

def _run_single_regularizer(
    log10_num_electrons, rixs, photoemission, regularizer_width, parameters
):
    """Run deconvolution for a single input regularization strength
    """
    impulse_response, pax_spectra, xray_xy = simulate_pax.simulate_from_presets(
        log10_num_electrons,
        rixs,
        photoemission,
        parameters["simulations"],
        parameters["energy_loss"],
    )
    deconvolver = deconvolvers.LRFisterDeconvolve(
        impulse_response["x"],
        impulse_response["y"],
        pax_spectra["x"],
        regularizer_width,
        iterations=parameters["iterations"],
        ground_truth_y=xray_xy["y"],
    )
    deconvolver.fit(np.array(pax_spectra["y"]))
    return deconvolver

def load(log10_num_electrons, rixs="schlappa", photoemission="ag"):
    """Load PAX simulation results
    """
    file_name = _get_filename(log10_num_electrons, rixs, photoemission)
    with open(file_name, "rb") as f:
        data = pickle.load(f)
    return data

def print_parameters(log10_num_electrons, rixs="schlappa", photoemission="ag"):
    """Load a PAX simulation and print some parameters it was run with
    """
    data = load(log10_num_electrons, rixs, photoemission)
    to_print = {
        "iterations": data["cv_deconvolver"].iterations,
        "cv_fold": data["cv_deconvolver"].cv_,
        "regularizer_widths": data["cv_deconvolver"].regularization_strengths,
        "shape of input PAX data": np.shape(data["pax_spectra"]["y"]),
    }
    pprint.pprint(to_print)

def _get_filename(log10_num_electrons, rixs, photoemission):
    file_name = "{}/{}_{}_rixs_1E{}.pickle".format(
        PROCESSED_DATA_DIR, photoemission, rixs, log10_num_electrons
    )
    return file_name


import os
import tarfile
import tempfile
from operator import add

import h5py
import pandas as pd
import wget

def get_name(index, hdf5_data):
    name = hdf5_data['/digitStruct/name']
    return ''.join([chr(v[0]) for v in hdf5_data[name[index][0]].value])

def get_bbox(index, hdf5_data):
    """Retrieves the bounding box from hdf5 data for a specific index.

    Args:
        index (int): index of image.
        hdf5_data (h5py file): h5py file containing bounding box information.

    Returns:
        dictionnary: label, left, top, width and height values for the bounding box.
    """
    attrs = {}
    item = hdf5_data['digitStruct']['bbox'][index].item()
    for key in ['label', 'left', 'top', 'width', 'height']:
        attr = hdf5_data[item][key]
        values = [int(hdf5_data[attr.value[i].item()].value[0][0])
                  for i in range(len(attr))] if len(attr) > 1 else [int(attr.value[0][0])]
        attrs[key] = values
    return attrs

def img_boundingbox_data_constructor(data_folder, mode, csv_path):
    """Creates bounding boxes for all images. This will generate a csv file indicating for each image the label and bounding box coordinates
    and return the corresponding DataFrame.

    Args:
        data_folder (string): path to data directory containing digitStruct.mat file.
        mode (string): training or testing.
        csv_path (string): path to save the csv file containing the bounding boxes information.

    Returns:
        DataFrame: bounding boxes information (image, label and coordinates)
    """
    f = h5py.File(os.path.join(data_folder, "digitStruct.mat"), 'r')
    row_list = []
    num_example = f['/digitStruct/bbox'].shape[0]
    logging_interval = num_example // 10
    print("found %d number of examples for %s" % (num_example, mode))
    for j in range(num_example):
        if j % logging_interval == 0:
            print("retrieving bounding box for %s: %.2f%%" % (mode, j / num_example * 100))
        img_name = get_name(j, f)
        bbox = get_bbox(j, f)
        row_dict = {
            'image': os.path.join(mode, img_name),
            'label': bbox["label"],
            'x1': bbox["left"],
            'y1': bbox["top"],
            'x2': list(map(add, bbox["left"], bbox["width"])),
            'y2': list(map(add, bbox["top"], bbox["height"]))
        }
        row_list.append(row_dict)
    bbox_df = pd.DataFrame(row_list, columns=['image', 'label', 'x1', 'y1', 'x2', 'y2'])
    bbox_df.to_csv(csv_path, index=False)
    return bbox_df

def load_data(path=None):
    """Downloads the svhn dataset to local storage, if not already downloaded. This will generate 2 csv files (train and test), which contain all the path
        information.

    Args:
        path (str, optional): The path to store the svhn data. Defaults to None, will save at `tempfile.gettempdir()`.

    Returns:
    string: path to train csv file.
    string: path to test csv file.
    string: path to data directory.
    """
    if path is None:
        path = os.path.join(tempfile.gettempdir(), ".fe", "SVHN")
    if not os.path.exists(path):
        os.makedirs(path)
    train_csv = os.path.join(path, "train_data.csv")
    test_csv = os.path.join(path, "test_data.csv")
    train_compressed_path = os.path.join(path, "train.tar.gz")
    test_compressed_path = os.path.join(path, "test.tar.gz")
    train_folder_path = os.path.join(path, "train")
    test_folder_path = os.path.join(path, "test")
    if not (os.path.exists(train_compressed_path) and os.path.exists(test_compressed_path)):
        print("Downloading data to %s" % path)
        wget.download('http://ufldl.stanford.edu/housenumbers/train.tar.gz', path)
        wget.download("http://ufldl.stanford.edu/housenumbers/test.tar.gz", path)
    if not (os.path.exists(train_folder_path) and os.path.exists(test_folder_path)):
        print(" ")
        print("extracting data...")
        with tarfile.open(train_compressed_path) as tar:
            tar.extractall(path)
        with tarfile.open(test_compressed_path) as tar:
            tar.extractall(path)
    if not (os.path.exists(train_csv) and os.path.exists(test_csv)):
        print("constructing bounding box data...")
        img_boundingbox_data_constructor(train_folder_path, "train", train_csv)
        img_boundingbox_data_constructor(test_folder_path, "test", test_csv)
    return train_csv, test_csv, path


import os.path
import json
from util import is_macos, is_darkmode

class Preferences:
    pref_file = os.path.join(os.getcwd(), "prefs", "config.json")
    defaults = os.path.join(os.getcwd(), "prefs", "defaults.json")
    copy_list = [
        'report_sheet',
        'format_unfinished',
        'unfinished_color',
        'class_portal',
        'protected_set',
        'debug'
    ]

    def __init__(self):
        with open(Preferences.defaults, "r+") as df:
            self.defaults = json.load(df)
        if not os.path.isfile(Preferences.pref_file):
            with open(Preferences.pref_file, 'w+') as pf:
                build_prefs = self.get_default_theme()
                for pref in Preferences.copy_list:
                    build_prefs[pref] = self.defaults[pref]
                json.dump(build_prefs, pf)
        with open(Preferences.pref_file, "r+") as pf:
            self.prefs = json.load(pf)
        self.pref_file = Preferences.pref_file

    def has_pref(self, key):
        return key in self.prefs

    def get_pref(self, key):
        if self.has_pref(key): return self.prefs[key]
        return None

    def update_pref(self, key, value):
        self.prefs[key] = value
        self.save_prefs()

    def save_prefs(self):
        with open(Preferences.pref_file, "w+") as pf:
            json.dump(self.prefs, pf)

    def get_default_pref(self, key):
        if key in self.defaults: return self.defaults[key]
        return None

    def get_default_theme(self):
        return self.defaults['dark_theme'] if is_darkmode() else self.defaults['light_theme']

prefs = Preferences()


import discord, datetime, time
from ..admin.managecommands import perms
import json
from discord.utils import get
from pymongo import MongoClient, collation
from discord.ext import commands, tasks
import time
import os
import pymongo as pm
import random

class Userinfo(commands.Cog):
    def __init__(self, bot):
        self.bot = bot

# Userinfo

    @commands.command(pass_context=True, aliases= ['userinfo', 'profile'])
    @commands.check(perms)
    async def user(self, ctx, member:discord.Member = None):
        if member is None:

            roles_list = ' | '.join(map(str, ctx.author.roles))

            embed=discord.Embed(title=member, color=random.randint(0, 0xFFFFFF))
            embed.add_field(name="ID", value=ctx.author.id, inline=False)
            embed.add_field(name="Nickname", value=ctx.author.nick, inline=False)
            embed.add_field(name="Highest role", value=ctx.author.top_role, inline=False)
            embed.add_field(name="Roles", value=roles_list, inline=False)
            embed.set_image(url=ctx.author.avatar_url)
            await ctx.reply(embed=embed)

        else:

            roles_list = ' | '.join(map(str, member.roles))

            embed=discord.Embed(title=member, color=random.randint(0, 0xFFFFFF))
            embed.add_field(name="ID", value=member.id, inline=False)
            embed.add_field(name="Nickname", value=member.nick, inline=False)
            embed.add_field(name="Highest role", value=member.top_role, inline=False)
            embed.add_field(name="Roles", value=roles_list, inline=False)
            embed.set_image(url=member.avatar_url)
            await ctx.reply(embed=embed)

    @commands.command(pass_context=True, aliases=['av'])
    @commands.check(perms)
    async def avatar(self, ctx, member:discord.Member = None):
        if member is None:

            embed=discord.Embed(title=ctx.author, color=random.randint(0, 0xFFFFFF))
            embed.set_image(url=ctx.author.avatar_url)
            await ctx.reply(embed=embed)

        else:

            embed=discord.Embed(title=member, color=random.randint(0, 0xFFFFFF))
            embed.set_image(url=member.avatar_url)
            await ctx.reply(embed=embed)

def setup(bot):
    bot.add_cog(Userinfo(bot))


from urllib.request import urlopen, HTTPError
from bs4 import BeautifulSoup
from urllib.parse import urljoin
from utils.parser import ParserHTML
from utils.data import saveDataOnFile, readDataFromFile

URL_SEED = 'https://pt.wikipedia.org/'
TODO_LIST = [URL_SEED]
DONE_LIST = []
BLACKLIST = set()
MAX_LINKS_VISITED = 100

parserHTML = ParserHTML()

def getInternalLinksFrom(url):
    regexInternalWikiLinks = r"^(/wiki/)(.*)?$"
    html = urlopen(url)
    parserHTML.loadPage(BeautifulSoup, html.read())
    return parserHTML.getTagA(regexInternalWikiLinks)

def isLinkNotAlreadyKnow(pulledLink):
    if 'href' in pulledLink.attrs:
        newURL = urljoin(URL_SEED, pulledLink.attrs['href'])
        if newURL not in DONE_LIST and newURL not in TODO_LIST:
            return True, newURL
    return False, None

def scrapper():
    counterVisitedLinks = 0

    while TODO_LIST:
        nextLinkToExplore = TODO_LIST.pop(0)
        if nextLinkToExplore not in DONE_LIST:
            try:
                if nextLinkToExplore in BLACKLIST:
                    continue

                newLinksList = getInternalLinksFrom(nextLinkToExplore)
                DONE_LIST.append(nextLinkToExplore)

                print(f'done: {nextLinkToExplore} - internal links: {len(newLinksList)}')

                for newLinkItem in newLinksList:
                    isNotAlreadyKnow, newUrl = isLinkNotAlreadyKnow(newLinkItem)

                    if isNotAlreadyKnow:
                        TODO_LIST.append(newUrl)
            except HTTPError as e:
                print(f'done: {nextLinkToExplore} - error: {e.errno}')
                BLACKLIST.add(nextLinkToExplore)

            counterVisitedLinks += 1

        if counterVisitedLinks == MAX_LINKS_VISITED:
            saveDataOnFile('TODO_LIST.txt', TODO_LIST)
            saveDataOnFile('DONE_LIST.txt', DONE_LIST)
            saveDataOnFile('BLACKLIST.txt', BLACKLIST)
            counterVisitedLinks = 0
            print('--> Data lists saved on files')

def start():
    global TODO_LIST
    global DONE_LIST
    global BLACKLIST

    TODO_LIST = readDataFromFile('TODO_LIST.txt') or TODO_LIST
    DONE_LIST = readDataFromFile('DONE_LIST.txt') or DONE_LIST
    BLACKLIST = readDataFromFile('BLACKLIST.txt') or BLACKLIST

    scrapper()

if __name__ == '__main__':
    try:
        print('--> Scraper running...')
        start()
    except KeyboardInterrupt:
        saveDataOnFile('TODO_LIST.txt', TODO_LIST)
        saveDataOnFile('DONE_LIST.txt', DONE_LIST)
        saveDataOnFile('BLACKLIST.txt', BLACKLIST)
        print('Bye bye!!')


import requests, urllib3
from config import Config

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

class UnauthorizedApiCall(Exception):
    pass

class ClientTmdbApi:
    url = "https://api.themoviedb.org/3/"
    cookie_tag = "Set-Cookie"
    session = requests.Session()
    session.headers["Accept"] = "application/json"
    session.headers["Authorization"] = Config.TMDB_API_CREDENTIALS

    def _request(self, api_method, json=None):
        if json is None:
            response = self.session.get(self.url + api_method, verify=False)
        else:
            response = self.session.post(self.url + api_method, json=json, verify=False)
            if response.status_code == 401:
                raise UnauthorizedApiCall()
        # required to avoid new session creation on every request
        if not self.session.headers.get(self.cookie_tag):
            self.session.headers[self.cookie_tag] = response.headers.get(
                self.cookie_tag
            )
        return response.json()

    def get_person_object(
        self, id_person=40239, object_name="movie_credits", language="cz-CZ"
    ):
        return self._request(
            "{}/{}/{}?language={}".format(
                "person", str(id_person), object_name, language
            )
        )

    def get_object_details(self, id_movie, object_name="movie"):
        return self._request("{}/{}".format(object_name, str(id_movie)))


import json
import logging
import os

import pytest
from librelingo_fakes import fakes
from librelingo_json_export.export import _export_skill

@pytest.fixture
def export_path():
    return fakes.path()

def test_creates_the_challenges_file(fs, export_path):
    random_name, fake_skill = fakes.get_fake_skill()
    _export_skill(export_path, fake_skill, fakes.course1)
    assert os.path.exists(export_path / "challenges" / f"animals-{random_name}.json")

def test_creates_the_introduction_file(fs, export_path):
    fake_name = fakes.fake_string()
    introduction = f"# *Hello* (https://example.com)[_{fake_name}_]!"
    random_name, fake_skill = fakes.get_fake_skill(introduction=introduction)
    _export_skill(export_path, fake_skill, fakes.course1)
    with open(
        export_path / "introduction" / f"animals-{random_name}.md"
    ) as animals_file:
        introduction_file_content = animals_file.read()
        assert introduction_file_content == introduction

def test_does_not_create_an_introduction_file_if_theres_no_introduction(
    fs, export_path
):
    random_name, fake_skill = fakes.get_fake_skill()
    _export_skill(export_path, fake_skill, fakes.course1)
    assert not os.path.exists(
        export_path / "introduction" / f"animals-{random_name}.md"
    )

@pytest.fixture
def mock_get_skill_data(mocker):
    return mocker.patch("librelingo_json_export.export._get_skill_data")

def test_calls__get_skill_data_with_correct_value(fs, export_path, mock_get_skill_data):
    mock_get_skill_data.return_value = []
    _export_skill(export_path, fakes.skillWithPhraseAndWord, fakes.course1)
    mock_get_skill_data.assert_called_with(fakes.skillWithPhraseAndWord, fakes.course1)

def test_writes_correct_value_into_json_file(fs, export_path, mock_get_skill_data):
    fake_skill_data = {"fake_skill_data": 1000}
    mock_get_skill_data.return_value = fake_skill_data
    _export_skill(export_path, fakes.skillWithPhraseAndWord, fakes.course1)
    with open(export_path / "challenges" / "masculine.json") as masculine_file:
        assert json.loads(masculine_file.read()) == fake_skill_data

def test_assert_logs_correctly(fs, caplog, export_path):
    with caplog.at_level(logging.INFO, logger="librelingo_json_export"):
        _, fake_skill = fakes.get_fake_skill()
        _export_skill(export_path, fake_skill, fakes.course1)
        assert caplog.record_tuples[0] == (
            "librelingo_json_export",
            logging.INFO,
            f"Writing skill '{fake_skill.name}'",
        )


from django.forms import ModelForm
from django.utils.translation import gettext_lazy as _
from .models import Profile, Tool
from django import forms
from django.contrib.auth.models import User

class ProfileForm(ModelForm):
    class Meta:
        model = Profile
        fields = ('cc_number', 'cc_exp', 'cc_fullname','zip_code', 'phone', 'profile_pic', 'terms_and_cond',)
        labels = {
            'cc_number': _('* Credit Card #:'),
            'cc_exp': _('* Credit Card Exp:'),
            'cc_fullname': _('* Full name on Credit Card:'),
            'zip_code': _('Zip Code:'),
            'profile_pic': _('Profile Image:'),
            'terms_and_cond': _('I agree to the terms and conditions:'),
        }
    def clean(self):
        super(ProfileForm, self).clean()
        tac = self.cleaned_data.get('terms_and_cond')
        if tac != True:
            self._errors['terms_and_cond'] = self.error_class(['You must agree to the terms and conditions to use this site!'])
        return self.cleaned_data

class ToolForm(ModelForm):
    class Meta:
        model = Tool
        fields = ('name', 'description', 'tool_value', 'for_sale', 'visible', 'tool_pic')
        labels = {
            'tool_value': _('Tool Value ($ USD)')
        }

class UserUpdateForm(ModelForm):
    email = forms.EmailField(
    max_length=100,
    required = True,
    widget=forms.TextInput(attrs={'class': 'form-control', 'placeholder': 'Email'}),
    )
    username = forms.CharField(
    max_length=100,
    required = True,
    widget=forms.TextInput(attrs={'class': 'form-control', 'placeholder': 'Username'}),
    )
    class Meta:
        model = User
        fields = ['username', 'email',]

class ProfileUpdateForm(ModelForm):
    class Meta:
        model = Profile
        fields = ('cc_number', 'cc_exp', 'cc_fullname','zip_code', 'phone', 'profile_pic',)
        labels = {
            'cc_number': _('* Credit Card #:'),
            'cc_exp': _('* Credit Card Exp:'),
            'cc_fullname': _('* Full name on Credit Card:'),
            'zip_code': _('Zip Code:'),
            'profile_pic': _('Profile Image:'),
        }


import os
from pathlib import Path
import logging

logging.basicConfig(level=logging.INFO)

project_name = "Currency_Converter_Chatbot"

list_of_files = [
    # ".github/workflows/.gitkeep",
    f"src/{project_name}/components.py",
    f"src/{project_name}/entity/config_entity.py",
    f"src/{project_name}/exception.py ",
    f"src/{project_name}/logger.py ",
    f"src/{project_name}/utils.py ",
    "app.py",
    "requirements.txt",
    "setup.py",
]

for filepath in list_of_files:
    filepath = Path(filepath)
    filedir, filename = os.path.split(filepath)

    if filedir != "":
        os.makedirs(filedir, exist_ok=True)
        logging.info(f"Creating directory:{filedir} for the file {filename}")

    if (not os.path.exists(filepath)) or (os.path.getsize(filepath) == 0):
        with open(filepath,'w') as f:
            pass
            logging.info(f"Creating empty file: {filepath}")

    else:
        logging.info(f"{filename} is already exists")


import unittest
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains

from helpers.base_test import BaseTest
from helpers.decorators import screenshot_decorator

class HoversPageTests(BaseTest):
    @screenshot_decorator
    def testHovers_whenAvatarIsHoveredOver_shouldCaptionAppearUnderTheImage(self):
        page_url = self.base_url + "hovers"
        self.driver.get(page_url)

        actions = ActionChains(self.driver)

        avatar = self.driver.find_elements(By.XPATH, '//*[@class="figure"]/img')[0]
        actions.move_to_element(avatar)
        actions.perform()

        caption = self.driver.find_elements(By.XPATH, '//*[@class="figcaption"]/h5')[0]
        caption_text = caption.text

        expected_caption_text = "name: user1"

        self.assertEqual(expected_caption_text, caption_text,
                         f"Expected paragraph text differ from actual paragraph text for page url:"
                         f" {self.driver.current_url}.")

if __name__ == '__main__':
    unittest.main()


import numpy as np
from random import random
from PIL import Image, ImageOps

import torch
import torch.utils.data as data
import torchvision

from scipy.spatial.distance import cdist

class BaselineDataset(data.Dataset):
    def __init__(self, fls, transforms=None):

        self.fls = fls
        self.clss = np.array([f.split('/')[-2] for f in fls])
        self.domains = np.array([f.split('/')[-3] for f in fls])
        self.transforms = transforms

    def __getitem__(self, item):

        sample_domain = self.domains[item]
        if sample_domain=='sketch' or sample_domain=='quickdraw':
            sample = ImageOps.invert(Image.open(self.fls[item])).convert(mode='RGB')
        else:
            sample = Image.open(self.fls[item]).convert(mode='RGB')

        clss = self.clss[item]

        if self.transforms is not None:
            sample = self.transforms(sample)

        return sample, clss

    def __len__(self):
        return len(self.fls)

class CuMixloader(data.Dataset):

    def __init__(self, fls, clss, doms, dict_domain, transforms=None):

        self.fls = fls
        self.clss = clss
        self.domains = doms
        self.dict_domain = dict_domain
        self.transforms = transforms

    def __getitem__(self, item):

        sample_domain = self.domains[item]
        if sample_domain=='sketch' or sample_domain=='quickdraw':
            sample = ImageOps.invert(Image.open(self.fls[item])).convert(mode='RGB')
        else:
            sample = Image.open(self.fls[item]).convert(mode='RGB')

        clss = self.clss[item]

        if self.transforms is not None:
            sample = self.transforms(sample)

        return sample, clss, self.dict_domain[sample_domain]

    def __len__(self):
        return len(self.fls)

class SAKELoader(data.Dataset):
    def __init__(self, fls, cid_mask, transforms=None):

        self.fls = fls
        self.cid_mask = cid_mask
        self.clss = np.array([f.split('/')[-2] for f in fls])
        self.domains = np.array([f.split('/')[-3] for f in fls])
        self.transforms = transforms

    def __getitem__(self, item):

        sample_domain = self.domains[item]
        if sample_domain=='sketch':
            sample = ImageOps.invert(Image.open(self.fls[item])).convert(mode='RGB')
        else:
            sample = Image.open(self.fls[item]).convert(mode='RGB')

        clss = self.clss[item]

        if self.transforms is not None:
            sample = self.transforms(sample)

        return sample, clss, self.cid_mask[clss]

    def __len__(self):
        return len(self.fls)

class SAKELoader_with_domainlabel(data.Dataset):
    def __init__(self, fls, cid_mask=None, transforms=None):

        self.fls = fls
        self.cid_mask = cid_mask
        self.clss = np.array([f.split('/')[-2] for f in fls])
        self.domains = np.array([f.split('/')[-3] for f in fls])
        self.transforms = transforms

    def __getitem__(self, item):

        sample_domain = self.domains[item]
        if sample_domain=='sketch':
            sample = ImageOps.invert(Image.open(self.fls[item])).convert(mode='RGB')
            domain_label = np.array([0])
        else:
            sample = Image.open(self.fls[item]).convert(mode='RGB')
            domain_label = np.array([1])

        clss = self.clss[item]

        if self.transforms is not None:
            sample = self.transforms(sample)

        if self.cid_mask is not None:
            return sample, clss, self.cid_mask[clss], domain_label
        else:
            return sample, clss, domain_label

    def __len__(self):
        return len(self.fls)

class Doodle2Search_Loader(data.Dataset):
    def __init__(self, fls_sketch, fls_image, semantic_vec, tr_classes, dict_clss, transforms=None):

        self.fls_sketch = fls_sketch
        self.fls_image = fls_image

        self.cls_sketch = np.array([f.split('/')[-2] for f in self.fls_sketch])
        self.cls_image = np.array([f.split('/')[-2] for f in self.fls_image])

        self.tr_classes = tr_classes
        self.dict_clss = dict_clss

        self.semantic_vec = semantic_vec
        # self.sim_matrix = np.exp(-np.square(cdist(self.semantic_vec, self.semantic_vec, 'euclidean'))/0.1)
        cls_euc = cdist(self.semantic_vec, self.semantic_vec, 'euclidean')
        cls_euc_scaled = cls_euc/np.expand_dims(np.max(cls_euc, axis=1), axis=1)
        self.sim_matrix = np.exp(-cls_euc_scaled)

        self.transforms = transforms

    def __getitem__(self, item):

        sketch = ImageOps.invert(Image.open(self.fls_sketch[item])).convert(mode='RGB')
        sketch_cls = self.cls_sketch[item]
        sketch_cls_numeric = self.dict_clss.get(sketch_cls)

        w2v = torch.FloatTensor(self.semantic_vec[sketch_cls_numeric, :])

        # Find negative sample
        possible_classes = self.tr_classes[self.tr_classes!=sketch_cls]
        sim = self.sim_matrix[sketch_cls_numeric, :]
        sim = np.array([sim[self.dict_clss.get(x)] for x in possible_classes])

        # norm = np.linalg.norm(sim, ord=1) # Similarity to probability
        # sim = sim/norm
        sim /= np.sum(sim)

        image_neg_cls = np.random.choice(possible_classes, 1, p=sim)[0]
        image_neg = Image.open(np.random.choice(self.fls_image[np.where(self.cls_image==image_neg_cls)[0]], 1)[0]).convert(mode='RGB')

        image_pos = Image.open(np.random.choice(self.fls_image[np.where(self.cls_image==sketch_cls)[0]], 1)[0]).convert(mode='RGB')

        if self.transforms is not None:
            sketch = self.transforms(sketch)
            image_pos = self.transforms(image_pos)
            image_neg = self.transforms(image_neg)

        return sketch, image_pos, image_neg, w2v

    def __len__(self):
        return len(self.fls_sketch)

class JigsawDataset(data.Dataset):
    def __init__(self, fls, transforms=None, jig_classes=30, bias_whole_image=0.9):

        self.fls = fls
        self.clss = np.array([f.split('/')[-2] for f in fls])
        self.domains = np.array([f.split('/')[-3] for f in fls])

        self.permutations = self.__retrieve_permutations(jig_classes)
        self.grid_size = 3
        self.bias_whole_image = bias_whole_image # biases the training procedure to show the whole image more often

        self._image_transformer = transforms['image']
        self._augment_tile = transforms['tile']

        def make_grid(x):
            return torchvision.utils.make_grid(x, self.grid_size, padding=0)
        self.returnFunc = make_grid

    def get_tile(self, img, n):
        w = float(img.size[0]) / self.grid_size
        y = int(n / self.grid_size)
        x = n % self.grid_size
        tile = img.crop([x * w, y * w, (x + 1) * w, (y + 1) * w])
        tile = self._augment_tile(tile)
        return tile

    def get_image(self, item):

        sample_domain = self.domains[item]
        if sample_domain=='sketch' or sample_domain=='quickdraw':
            sample = ImageOps.invert(Image.open(self.fls[item])).convert(mode='RGB')
        else:
            sample = Image.open(self.fls[item]).convert(mode='RGB')
        return self._image_transformer(sample)

    def __getitem__(self, item):

        img = self.get_image(item)
        n_grids = self.grid_size ** 2
        tiles = [None] * n_grids
        for n in range(n_grids):
            tiles[n] = self.get_tile(img, n)

        order = np.random.randint(len(self.permutations) + 1) # added 1 for class 0: unsorted
        if self.bias_whole_image:
            if self.bias_whole_image > random():
                order = 0

        if order == 0:
            data = tiles
        else:
            data = [tiles[self.permutations[order - 1][t]] for t in range(n_grids)]

        data = torch.stack(data, 0)
        data = self.returnFunc(data)

        return torch.cat([self._augment_tile(img), data], 0), order, self.clss[item]

    def __len__(self):
        return len(self.fls)

    def __retrieve_permutations(self, classes):
        all_perm = np.load(os.path.join(_BASE_PATH, 'permutations_%d.npy' % (classes)))
        # from range [1,9] to [0,8]
        if all_perm.min() == 1:
            all_perm = all_perm - 1

        return all_perm


import logging

from django.core.mail import send_mail

# Create your views here.
from rest_framework.response import Response
from rest_framework.views import APIView
from rest_framework.generics import CreateAPIView

from .models import LandingModel, ManagersModel, FormBlockAdminModel
from .serializers import LandingModelAdminSerializer, FormBlockSerializer
from .tasks import send_email

logger = logging.getLogger('main')

class LandingPageFormView(CreateAPIView):
    """View to send a form"""
    queryset = FormBlockAdminModel.objects.all()

    def perform_create(self, serializer):
        try:
            mail_to = ManagersModel.objects.all().values()
            data = self.request.data
            name = data["form_customer_name"]
            phone = data["form_customer_phone"]
            message = data["form_customer_phone"]
            for recipient in mail_to:
                send_to = recipient['managers_email']
                send_email.delay(send_to, name, phone, message)
            serializer.save()
        except Exception as e:
            logger.warning(e)

class LandingPageView(APIView):
    """View to get landing context"""

    def get(self, request, *args, **kwargs):
        try:
            queryset = LandingModel.objects.latest('id')
            serializer = LandingModelAdminSerializer(queryset, context={'request': request})
            return Response(serializer.data)
        except Exception as e:
            logger.warning(e)


import math

import torch
import torch.nn as nn
import numpy as np
from utils.diffusion_util import GLD

class DualDiffusion(nn.Module):
    def __init__(self, config, device, target_dim=96):
        super().__init__()
        self.device = device
        self.target_dim = target_dim

        self.emb_time_dim = config["model"]["timeemb"]
        self.emb_feature_dim = config["model"]["featureemb"]
        self.is_unconditional = config["model"]["is_unconditional"]
        self.finger_feature_embedding = nn.Linear(3, 3) # 3->3

        self.emb_total_dim = self.emb_time_dim + self.emb_feature_dim
        if not self.is_unconditional:
            self.emb_total_dim += 1  # for conditional mask
        self.embed_layer = nn.Embedding(
            num_embeddings=self.target_dim, embedding_dim=self.emb_feature_dim
        )

        config_diff = config["diffusion"]
        config_diff["side_dim"] = self.emb_total_dim

        input_dim = 1 if self.is_unconditional else 2
        self.diffmodel = GLD(config_diff, input_dim)

        # parameters for diffusion models
        self.num_steps = config_diff["num_steps"]
        if config_diff["schedule"] == "quad":
            self.beta = np.linspace(
                config_diff["beta_start"] ** 0.5, config_diff["beta_end"] ** 0.5, self.num_steps
            ) ** 2
        elif config_diff["schedule"] == "linear":
            self.beta = np.linspace(
                config_diff["beta_start"], config_diff["beta_end"], self.num_steps
            )
        elif config_diff["schedule"] == "cosine":
            self.beta = self.betas_for_alpha_bar(
                self.num_steps,
                lambda t: math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2,
            )

        self.alpha_hat = 1 - self.beta
        self.alpha = np.cumprod(self.alpha_hat)
        self.alpha_torch = torch.tensor(self.alpha).float().to(self.device).unsqueeze(1).unsqueeze(1)

    def betas_for_alpha_bar(self, num_diffusion_timesteps, alpha_bar, max_beta=0.5):
        # """
        # Create a beta schedule that discretizes the given alpha_t_bar function,
        # which defines the cumulative product of (1-beta) over time from t = [0,1].
        # :param num_diffusion_timesteps: the number of betas to produce.
        # :param alpha_bar: a lambda that takes an argument t from 0 to 1 and
        #                   produces the cumulative product of (1-beta) up to that
        #                   part of the diffusion process.
        # :param max_beta: the maximum beta to use; use values lower than 1 to
        #                  prevent singularities.
        # """
        betas = []
        for i in range(num_diffusion_timesteps):
            t1 = i / num_diffusion_timesteps
            t2 = (i + 1) / num_diffusion_timesteps
            betas.append(min(1 - alpha_bar(t2) / alpha_bar(t1), max_beta))
        return np.array(betas)

    def time_embedding(self, pos, d_model=128):
        pe = torch.zeros(pos.shape[0], pos.shape[1], d_model).to(self.device)
        position = pos.unsqueeze(2)
        div_term = 1 / torch.pow(
            10000.0, torch.arange(0, d_model, 2).to(self.device) / d_model
        )
        pe[:, :, 0::2] = torch.sin(position * div_term)
        pe[:, :, 1::2] = torch.cos(position * div_term)
        return pe

    def get_side_info(self, observed_tp, cond_mask):
        B, K, L = cond_mask.shape

        time_embed = self.time_embedding(observed_tp, self.emb_time_dim)  # (B,L,emb)
        time_embed = time_embed.unsqueeze(2).expand(-1, -1, K, -1)
        feature_embed = self.embed_layer(
            torch.arange(self.target_dim).to(self.device)
        )  # (K,emb)
        feature_embed = feature_embed.unsqueeze(0).unsqueeze(0).expand(B, L, -1, -1)

        side_info = torch.cat([time_embed, feature_embed], dim=-1)  # (B,L,K,*)
        side_info = side_info.permute(0, 3, 2, 1)  # (B,*,K,L)

        if not self.is_unconditional:
            side_mask = cond_mask.unsqueeze(1)  # (B,1,K,L)
            side_info = torch.cat([side_info, side_mask], dim=1)

        return side_info

    def calc_loss_valid(
            self, observed_data, cond_mask, side_info, is_train,observed_data_latent
    ):
        loss_sum = 0
        for t in range(self.num_steps):  # calculate loss for all t
            loss = self.calc_loss(
                observed_data, cond_mask, side_info, is_train,observed_data_latent, set_t=t
            )
            loss_sum += loss.detach()
        return loss_sum / self.num_steps

    def calc_loss(
            self, observed_data, cond_mask, side_info, is_train, observed_data_latent, set_t=-1
    ):
        B, K, L = observed_data.shape
        if is_train != 1:  # for validation
            t = (torch.ones(B) * set_t).long().to(self.device)
        else:
            t = torch.randint(0, self.num_steps, [B]).to(self.device)
        current_alpha = self.alpha_torch[t].to(self.device)  # (B,1,1)
        noise = torch.randn_like(observed_data).to(self.device)
        noisy_data = (current_alpha ** 0.5) * observed_data + (1.0 - current_alpha) ** 0.5 * noise

        total_input = self.set_input_to_diffmodel(noisy_data, observed_data, cond_mask)

        predicted = self.diffmodel(total_input, side_info, t)  # (B,K,L)

        target_mask = 1 - cond_mask
        residual = (noise - predicted) * target_mask
        num_eval = target_mask.sum()
        loss = (residual ** 2).sum() / (num_eval if num_eval > 0 else 1)

        ####################################

        noise_latent = torch.randn_like(observed_data_latent).to(self.device)
        noisy_data_latent = (current_alpha ** 0.5) * observed_data_latent + (1.0 - current_alpha) ** 0.5 * noise_latent

        total_input_latent = self.set_input_to_diffmodel(noisy_data_latent, observed_data_latent, cond_mask)

        predicted_latent = self.diffmodel(total_input_latent, side_info, t)  # (B,K,L)

        target_mask = 1 - cond_mask
        residual_latent = (noise_latent - predicted_latent) * target_mask
        num_eval = target_mask.sum()
        loss_latent = (residual_latent ** 2).sum() / (num_eval if num_eval > 0 else 1)
        return loss+loss_latent

    def set_input_to_diffmodel(self, noisy_data, observed_data, cond_mask):
        if self.is_unconditional:
            total_input = noisy_data.unsqueeze(1)  # (B,1,K,L)
        else:
            cond_obs = (cond_mask * observed_data).unsqueeze(1)
            noisy_target = ((1 - cond_mask) * noisy_data).unsqueeze(1)
            total_input = torch.cat([cond_obs, noisy_target], dim=1)  # (B,2,K,L)

        return total_input

    def impute(self, observed_data, cond_mask, side_info, n_samples):
        B, K, L = observed_data.shape

        imputed_samples = torch.zeros(B, n_samples, K, L).to(self.device)

        for i in range(n_samples):
            # generate noisy observation for unconditional model
            if self.is_unconditional:
                noisy_obs = observed_data
                noisy_cond_history = []
                for t in range(self.num_steps):
                    noise = torch.randn_like(noisy_obs)
                    noisy_obs = (self.alpha_hat[t] ** 0.5) * noisy_obs + self.beta[t] ** 0.5 * noise
                    noisy_cond_history.append(noisy_obs * cond_mask)

            current_sample = torch.randn_like(observed_data)

            for t in range(self.num_steps - 1, -1, -1):
                if self.is_unconditional:
                    diff_input = cond_mask * noisy_cond_history[t] + (1.0 - cond_mask) * current_sample
                    diff_input = diff_input.unsqueeze(1)  # (B,1,K,L)
                else:
                    cond_obs = (cond_mask * observed_data).unsqueeze(1)
                    noisy_target = ((1 - cond_mask) * current_sample).unsqueeze(1)
                    diff_input = torch.cat([cond_obs, noisy_target], dim=1)  # (B,2,K,L)
                predicted = self.diffmodel(diff_input, side_info, torch.tensor([t]).to(self.device))

                coeff1 = 1 / self.alpha_hat[t] ** 0.5
                coeff2 = (1 - self.alpha_hat[t]) / (1 - self.alpha[t]) ** 0.5
                current_sample = coeff1 * (current_sample - coeff2 * predicted)

                if t > 0:
                    noise = torch.randn_like(current_sample)
                    sigma = (
                                    (1.0 - self.alpha[t - 1]) / (1.0 - self.alpha[t]) * self.beta[t]
                            ) ** 0.5
                    current_sample += sigma * noise

            imputed_samples[:, i] = (current_sample * (1 - cond_mask) + observed_data * cond_mask).detach()
        return imputed_samples

    def forward(self, batch, is_train=1):
        (observed_data,observed_tp,gt_mask,observed_data_latent) = self.process_data(batch)
        cond_mask = gt_mask

        side_info = self.get_side_info(observed_tp, cond_mask)

        loss_func = self.calc_loss if is_train == 1 else self.calc_loss_valid

        return loss_func(observed_data, cond_mask, side_info, is_train, observed_data_latent)

    def evaluate(self, batch, n_samples):
        (
            observed_data,
            observed_tp,
            gt_mask,
            observed_data_latent
        ) = self.process_data(batch)
        # print(np.shape(observed_data))
        # print(np.shape(observed_data))
        # print(np.shape(observed_data))
        pose_t = batch["pose"].to(self.device).float()
        batch_size, total_input_n , joints_n = pose_t.size()
        temp_input_n=10
        temp_output_n = total_input_n-temp_input_n

        with torch.no_grad():
            pose_t_output = pose_t[:, temp_input_n:, :]
            pose_t_output = pose_t_output.view(-1, temp_output_n, 21, 3).cpu().numpy()
            cond_mask = gt_mask
            target_mask = 1 - cond_mask

            side_info = self.get_side_info(observed_tp, cond_mask)

            samples = self.impute(observed_data, cond_mask, side_info, n_samples)
            # samples= samples.permute(0, 2, 1)
            samples_latent = self.impute(observed_data_latent, cond_mask, side_info, n_samples)
            # samples_latent = samples_latent.permute(0, 2, 1)
            samples_mean = np.mean(samples.cpu().numpy(), axis=1)
            samples_latent_mean = np.mean(samples_latent.cpu().numpy(), axis=1)
            samples_mean_output = samples_mean[:,:,temp_input_n:]
            samples_mean_output = samples_mean_output.transpose(0, 2, 1).reshape(-1, temp_output_n, 21, 3)
            # print(np.shape(samples_mean_output))
            samples_latent_mean_output = samples_latent_mean[:, :, temp_input_n:]
            samples_latent_mean_output = samples_latent_mean_output.transpose(0, 2, 1).reshape(-1, temp_output_n, 21, 3)

            best_samples = samples
            # print(np.shape(pose_t_output), np.shape(samples_mean_output))
            if self.mpjpe_error(torch.from_numpy(samples_mean_output),torch.from_numpy(pose_t_output)) > self.mpjpe_error(torch.from_numpy(samples_latent_mean_output),torch.from_numpy(pose_t_output)):
                best_samples = samples_latent
            samples =best_samples
            # trenorm_pose.view(-1, output_n, args.joints, 3)
        # print(224, np.shape(samples),'post truth',np.shape(pose_t),'mean',np.shape(samples_mean)) #torch.Size([20, 5, 63, 15]) post truth torch.Size([20, 15, 63]) mean (20, 63, 15)

        return samples, observed_data, target_mask, observed_tp

    def process_data(self, batch):
        latent_finger_63 = self.finger_features_process_data(batch)
        pose_latent = latent_finger_63 # [32, 15, 66]
        pose = batch["pose"].to(self.device).float() # [32, 15, 66]
        tp = batch["timepoints"].to(self.device).float()
        mask = batch["mask"].to(self.device).float()
        # print(216, np.shape(pose_t),np.shape(pose), np.shape(tp), np.shape(mask))

        pose_latent = pose_latent.permute(0, 2, 1)
        pose = pose.permute(0, 2, 1)
        mask = mask.permute(0, 2, 1)

        return (
            pose,
            tp,
            mask,
            pose_latent
        )
    def finger_features_process_data(self,batch):
        jt_idx = [[0,1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16], [17, 18, 19, 20]]
        pose = batch["pose"].to(self.device).float()
        batch_sise, input_feature_n, dimension_joints = pose.size()
        reshape_pose = pose.reshape(batch_sise, input_feature_n, 21, 3)
        finger = []
        for i in range(0,5):
            temp_reshape_pose = self.finger_feature_embedding(reshape_pose[:, :, jt_idx[i], :])
            finger.append(temp_reshape_pose) #[32, 15, 4, 3]

        latent_finger = torch.cat([finger[0], finger[1],finger[2],finger[3],finger[4]], dim=2)
        reshaped_latent_finger = torch.reshape(latent_finger, (batch_sise, input_feature_n, -1))
        # latent_finger_63 = self.fc1(reshaped_latent_finger)
        latent_finger_63 = reshaped_latent_finger

        # print(243, np.shape(latent_finger_63)) #[32, 15, 63]
        return  latent_finger_63

    def mpjpe_error(self, batch_imp, batch_gt):
        batch_imp = batch_imp.contiguous().view(-1, 3)
        batch_gt = batch_gt.contiguous().view(-1, 3)

        return torch.mean(torch.norm(batch_gt - batch_imp, 2, 1))

# import torch
# from thop import profile
#
# if __name__ == '__main__':
#     config = {
#         'train':
#             {
#                 'epochs': 100,
#                 'batch_size': 32,
#                 'batch_size_test': 32,
#                 'lr': 1.0e-3
#             },
#         'diffusion':
#             {
#                 'layers': 12,
#                 'channels': 64,
#                 'nheads': 8,
#                 'diffusion_embedding_dim': 128,
#                 'beta_start': 0.0001,
#                 'beta_end': 0.5,
#                 'num_steps': 50,
#                 'schedule': "cosine"
#             },
#         'model':
#             {
#                 'is_unconditional': 0,
#                 'timeemb': 128,
#                 'featureemb': 16
#             }
#     }
#     device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
#     print('Using device: %s' % device)
#
#     model = ModelMain(config, device, target_dim=(21 * 3)).to(device)
#     torch.Size([32, 15, 63])
#     torch.Size([32, 15])
#     torch.Size([32, 15, 63])
#
#     pose = torch.zeros(32,15,63).to(device)
#     mask=  torch.zeros(32,15,63).to(device)
#     timepoints = torch.zeros(32,15).to(device)
#     input_n= 75
#     s = {
#         "pose":pose[:, :input_n + 5],
#         "mask": mask[:, :input_n + 5],
#         "timepoints": timepoints[:, :input_n + 5]
#     }
#     print(271, np.shape(pose[:, :input_n + 5]))
#
#     output = model.evaluate(s, 5)
#     samples, _, eval_impute, _ = output
#     print(274, np.shape(samples))
# # #############################################
#     input_tensor = torch.randn(1, 10, 63)  # Example input tensor
#
#     flops, params = profile(model, inputs=(s,))
#     print(f"FLOPs: {flops}, Parameters: {params}")
#     total_params = sum(p.numel() for p in model.parameters())
#     print(f"Total Parameters: {total_params}")


import streamlit as st
import string
from nltk.tokenize import word_tokenize
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.feature_extraction.text import TfidfVectorizer
from datetime import datetime
import Stemmer                              # Descargar PyStemmer, pip install PyStemmer
import pandas as pd
import plotly.express as px
import plotly.figure_factory as plt
import joblib

# Funcion que cuenta el numero de fichero que incluye una lista
def contar_ficheros(lista_ficheros: list):
    return len(lista_ficheros)

# Funcion que convierte a minusculas el texto pasado como argumento y lo tokeniza/separa las palabras del texto
def tokenizar_texto(texto):
    lista_tokens = word_tokenize(str(texto.read(), encoding="utf8").lower(), language="spanish")
    return lista_tokens

# Funcion en la que aplicamos un lista de parada, es decir, le pasamos un fichero con las palabras o tokens que nos
# interesa eliminar del texto procesado y las quitamos
def limpiar_texto(lista_tokens: list):
    palabras = []
    # Abrimos nuestra lista de parada
    fichero_parada = open("Lista_Stop_Words.txt", "r", encoding="utf8")
    # Hemos leido el fichero como un string y necesitamos una lista de strings con las palabras a eliminar,
    # para ello aplicamos la funcion split para separar el texto por los espacios
    lista_parada = fichero_parada.read().split("\n")
    # Creamos una lista con los diferentes caracteres(!"#$%&'()*+, -./:;<=>?@[\]^_`{|}~) y los añadimos a nuestra
    # lista de parada, pues no nos son relevantes tampoco a la hora de aplicar nuestro algoritmo de aprendizaje
    puntuacion = list(string.punctuation)
    lista_parada += puntuacion
    # Aplicamos la lista de parada y nos quedamos solo con las palabras/tokens que nos interesan
    for palabra in lista_tokens:
        if palabra not in lista_parada:
            palabras.append(palabra)
    return palabras

# Funcion con la que aplicaremos el stemming (esta técnica nos ayudará a simplificar o reducir los tokens o palabras al stem)
def stemming(lista_palabras: list):
    texto = ""
    # Cogemos el stemmer de snowball en español
    stemmer = Stemmer.Stemmer('spanish')
    for palabra in lista_palabras:
        # Cogemos el stem de cada palabra
        s = stemmer.stemWord(palabra)
        texto = texto + " " + s
    return texto

# Funcion que genera una coleccion con los textos procesados, es decir, aplicando la tokenizacion, la transformacion a
# minusculas, eliminar las palabras que no se quieren a partir de una lista de parada y aplicacion de un stemmer(Snowball)
@st.cache
def generar_coleccion(lista_textos: list):
    coleccion = []
    for texto in lista_textos:
        # Aplicamos tokenizacion del texto
        tokens = tokenizar_texto(texto)
        # Aplicamos lista de parada
        lista_limpia = limpiar_texto(tokens)
        # Aplicamos stemmer
        texto = stemming(lista_limpia)
        coleccion.append(texto)
    return coleccion

def seleccionar_algoritmo(algoritmo: str):
    model = ""
    if algoritmo == "Gradient Boosted Tree":
        model = GradientBoostingClassifier()
    elif algoritmo == "Support Vector Machine":
        model = SVC()
    elif algoritmo == "Arbol Decision":
        model = DecisionTreeClassifier()
    return model

# Funcion que asigna el tipo de noticia (odio o no odio) a la que pertenece el texto
@st.cache
def asociar_clase(odio: list, no_odio: list):
    clase_odio = []
    clase_no_odio = []
    for texto_odio in odio:
        clase_odio.append('Odio')
    for texto_no in no_odio:
        clase_no_odio.append('No Odio')
    return clase_odio + clase_no_odio

def precision(tp: int, fp: int):
    return tp/(tp + fp)

def recall(tp: int, fn: int):
    return tp/(tp + fn)

def entrenar_modelo(algoritmo: str, coleccion_documentos: list, clases: list):
    tf = TfidfVectorizer()
    matriz_idf = tf.fit_transform(coleccion_documentos).toarray()
    df = pd.DataFrame(matriz_idf , columns=tf.get_feature_names_out())
    clf = seleccionar_algoritmo(algoritmo)
    X_train, X_test, Y_train, Y_test = train_test_split(df, clases, test_size=0.3)

    X_train = pd.get_dummies(X_train)
    X_test = pd.get_dummies(X_test)
    modelo = clf.fit(X_train, Y_train)
    y_pred = modelo.predict(X_test)
    tn, fp, fn, tp = confusion_matrix(Y_test, y_pred).ravel()
    cm = [[tn, fp], [fn, tp]]
    col1, col2, col3 = st.columns(3)
    with col1:
        st.metric("Precision: ", "{:.2f}".format((precision(tp, fp) * 100)) + "%", delta="{:.2f}".format((precision(tp, fp) * 100) - 95) + "%")
    with col2:
        st.metric("Accuracy: ", "{:.2f}".format(modelo.score(X_test, Y_test) * 100) + "%", delta="{:.2f}".format((modelo.score(X_test, Y_test) * 100) - 95) + "%")
    with col3:
        st.metric("Recall: ", "{:.2f}".format((recall(tp, fn) * 100)) + "%", delta="{:.2f}".format((recall(tp, fn) * 100) - 95) + "%")
    st.write("Resultados Entrenamiento: ")
    figura = plt.create_annotated_heatmap(cm, x=['Verdadero No Odio', 'Verdadero Odio'], y=['Predicho No Odio', 'Predicho Odio'], colorscale='Viridis')
    st.plotly_chart(figura)
    joblib.dump(tf.vocabulary_, 'vocabulario.bin')
    return modelo

def visualizacion_previa(odio, no_odio, algoritmo):
    num_odio = contar_ficheros(odio)
    num_no_odio = contar_ficheros(no_odio)

    ejemplares = {'Odio': num_odio, 'No Odio': num_no_odio}
    st.text_area("Vista Previa", "Ejemplares 'Odio': " + "\t" + str(num_odio) + "\nEjemplares 'No Odio': " + "\t" + str(num_no_odio) + "\nTotal: " + "\t" + str(num_odio + num_no_odio) + "\nAlgoritmo Seleccionado: " + "\t" + algoritmo, height=130)

    df = pd.DataFrame.from_dict(ejemplares, orient='index', columns=['Ejemplares'])
    df = df.rename_axis('Clase')
    figura = px.bar(df, y="Ejemplares", color="Ejemplares")
    st.plotly_chart(figura)


from django.db import models

class ShowManager(models.Manager):
    def basic_validator(self, postData):
        error = {}
        if len(postData['name']) < 2:
            errors["name"] = "Name must be at least 2 characters"
        if len(postData['network']) < 3:
            errors["network"] = "Network must be at least 3 characters"
        if len(postData['description']) < 2:
            errors["description"] = "Name must be at least 10 characters"
        return errors

class Show(models.Model):
    title = models.CharField(max_length=255)
    network = models.CharField(max_length=45)
    release_date = models.DateTimeField()
    description = models.CharField(max_length=255)
    created_at = models.DateTimeField(auto_now=True)
    updated_at = models.DateTimeField(auto_now_add=True)
    objects = ShowManager()