"""
Browser (:mod:`pydent.browser`)
=================================

.. versionadded:: 0.1
    Browser class created

.. currentmodule:: pydent.browser

Browser class for searching and cacheing results.
"""
import re
from collections import OrderedDict
from difflib import get_close_matches
from pprint import pformat
from typing import Callable
from typing import Dict
from typing import List
from typing import Union

import networkx as nx

from pydent import models as pydent_models
from pydent.base import ModelBase
from pydent.exceptions import ForbiddenRequestError
from pydent.exceptions import TridentBaseException
from pydent.interfaces import QueryInterface
from pydent.interfaces import QueryInterfaceABC
from pydent.marshaller import ModelRegistry
from pydent.models import Sample
from pydent.relationships import BaseRelationship
from pydent.sessionabc import SessionABC
from pydent.utils import logger
from pydent.utils.logging_helpers import did_you_mean

# TODO: browser documentation
# TODO: examples in sphinx
# TODO: methods to help pull relevant data from plans (user specifies types of data to
#       pull, and trident should pull and cache in the most efficient way possible)

class BrowserException(TridentBaseException):
    """Generic browser exception."""

class Browser(QueryInterfaceABC):
    """A class for browsing models and Aquarium inventory."""

    # TODO: ability to block model callbacks to enforce cache

    INTERFACE_CLASS = QueryInterface
    ACCEPTED_GET_RELATION_TYPES = [
        "HasOne",
        "HasMany",
        "HasManyThrough",
        "HasManyGeneric",
    ]

    def __init__(self, session: SessionABC, inherit_models: bool = False):
        """Instantiates a new browser from a AqSession instance.

        .. versionchanged:: 0.1.5a7
            'inherit_models' argument will inherit the sessions model_cache
            (default: False)

        :param session: a session instance
        :param inherit_models: if True, the browser will inherit the cache in the
            provided session's browser model_cache
        :type session: SessionABC
        """
        self.session = session
        self._list_models_fxn = self.sample_list
        self.use_cache = True
        self.model = Sample
        self.model_list_cache = {}
        self.model_cache = {}
        self.log = logger(name="Browser@{}".format(session.url))
        if session.browser and inherit_models:
            self.update_cache(session.browser.models)

    @property
    def model_name(self):
        return self.model.__name__

    # TODO: change session interface (find, where, etc.) to use cache IF use_
    #       cache = True
    # TODO: where and find queries can sort through models much more quickly than
    #       Aquarium, but can fallback to Aq

    @property
    def models(self):
        models = []
        for v in self.model_cache.values():
            models += list(v.values())
        return models

    def set_model(self, model_name):
        """Sets the default model of this browser."""
        self.model = ModelRegistry.get_model(model_name)
        if model_name == "Sample":
            self._list_models_fxn = self.sample_list
        else:
            self._list_models_fxn = self._generic_list_models

    def interface(self, model_class=None):
        """Returns a new model query interface.

        :param model_class:
        :type model_class: basestring
        :return: Interface
        :rtype: QueryInterface
        """
        if model_class is None:
            model_class = self.model_name
        return self.session.model_interface(
            model_class, interface_class=self.INTERFACE_CLASS
        )

    def _generic_list_models(self, **query):
        models = self.where(query, self.model_name)
        return ["{}: {}".format(m.id, m.name) for m in models]

    def sample_list(self, sample_type_id=None):
        """Returns a sample list."""
        path = "sample_list"
        if sample_type_id is not None:
            path += "/" + str(sample_type_id)
        return self.session.utils.aqhttp.get(path)

    def clear(self):
        """Clears the model cache."""
        self.model_list_cache = {}
        self.model_cache = {}

    def list_models(self, *args, **kwargs):
        def get_models():
            return self._list_models_fxn(*args, **kwargs)

        if self.use_cache:
            models_cache = self.model_list_cache.get("models", {})
            if self.model_name in models_cache:
                model_list = models_cache[self.model_name]
            else:
                model_list = get_models()
        else:
            model_list = get_models()
        self.model_list_cache.setdefault("models", {})
        self.model_list_cache["models"][self.model_name] = model_list[:]
        return model_list

    def where(
        self,
        query,
        model_class=None,
        primary_key="id",
        sample_type=None,
        methods=None,
        opts: Dict = None,
        page_size: int = None,
        include: Dict = None,
    ):
        """Perform a 'where' query. If models are found in the browser cache,
        those are returned, else new http queries are made to find the models.

        :param query: query as a dictionary
        :param model_class: model class to use (str)
        :param primary_key: which primary key to use (default: 'id')
        :param sample_type: optional sample_type short cut for finding samples
        :param kwargs: other kwargs
        :return: returned model list
        """
        if model_class is None:
            model_class = self.model_name
        if sample_type is not None:
            sample_type_id = self.find_by_name(sample_type, "SampleType").id
            query.update({"sample_type_id": sample_type_id})
        if self.use_cache and not methods and page_size is None:
            return self.cached_where(
                query,
                model_class,
                primary_key=primary_key,
                include=include,
                methods=methods,
                opts=opts,
            )
        else:
            return self.server_where(
                query,
                model_class,
                opts=opts,
                include=include,
                methods=methods,
                page_size=page_size,
            )

    def __query_helper(
        self,
        fname,
        query,
        model_class,
        sample_type=None,
        opts=None,
        params=None,
        as_single=False,
    ):
        """Builds a custom query for the browser.

        :param fname: the function name
        :type fname: basestring
        :param query: the query
        :type query: dict
        :param model_class: the name of the model class (e.g. "Sample")
        :type model_class: basestring
        :param sample_type: optional sample_type name
        :type sample_type: basestring
        :param opts: options to send to the function
        :type opts: dict
        :param params: additionaly keyword arguments to send to the function
        :type params: dict
        :param as_single: if True, will return the first model of the array or None if
            array is empty
        :type as_single: bool
        :return: Aquarium model or list of Aquarium models
        :rtype: ModelBase | list
        """
        if model_class is None:
            model_class = self.model_name
        if query is None:
            query = dict()
        if params is None:
            params = dict()
        if sample_type is not None:
            query.update(
                {"sample_type_id": self.find_by_name(sample_type, "SampleType").id}
            )
        interface = self.interface(model_class)
        fxn = getattr(interface, fname)
        if fname == "all":
            models = fxn(opts=opts, **params)
        else:
            models = fxn(query=query, opts=opts, **params)
        if as_single:
            models = [models]
        return self.update_cache(models).get(model_class, [])

    def one(self, model_class=None, sample_type=None, query=None, opts=None):
        """Finds one instance of a model (or returns None)

        :param model_class: the name of the model class (e.g. "Sample")
        :type model_class: basestring
        :param sample_type: optional sample_type name
        :type sample_type: basestring
        :param query: additional query to filter models
        :type query: dict
        :param opts: additional options
        :type opts: dict
        :return:
        :rtype:
        """
        models = self.__query_helper(
            "one", query, model_class, sample_type, opts=opts, as_single=True
        )
        if not models:
            return None
        return models[0]

    def last(self, num=1, model_class=None, sample_type=None, query=None):
        """Finds last models. Will NOT return cached models.

        :param num: number of models to return
        :type num: int
        :param model_class: the name of the model class (e.g. "Sample")
        :type model_class: basestring
        :param sample_type: optional sample_type name
        :type sample_type: basestring
        :param query: additional query to filter models
        :type query: dict
        :return:
        :rtype:
        """
        return self.__query_helper(
            "last", query, model_class, sample_type, params=dict(num=num)
        )

    def first(self, num=1, model_class=None, sample_type=None, query=None):
        """Finds first models. Will NOT return cached models.

        :param num: number of models to return
        :type num: int
        :param model_class: the name of the model class (e.g. "Sample")
        :type model_class: basestring
        :param sample_type: optional sample_type name
        :type sample_type: basestring
        :param query: additional query to filter models
        :type query: dict
        :return:
        :rtype:
        """
        return self.__query_helper(
            "first", query, model_class, sample_type, params=dict(num=num)
        )

    def find(self, model_id, model_class=None):
        """Finds a model by id. Will returned cached model if possible.

        :param model_id: model_id
        :type model_id: int
        :param model_class: the name of the model class (e.g. "Sample")
        :type model_class: basestring
        :return:
        :rtype:
        """
        if model_class is None:
            model_class = self.model_name
        if self.use_cache:
            return self.cached_find(model_class, model_id)
        return self.interface(model_class).find(model_id)

    def find_by_name(self, name, model_class=None, primary_key="id"):
        """Find model by name. Will return cached model if possible.

        :param name: name of the model
        :param model_class: the name of the model class (e.g. "Sample")
        :type model_class: basestring
        :param primary_key:
        :return:
        """
        models = self.where({"name": name}, model_class, primary_key=primary_key)
        if not models:
            return None
        return models[0]

    def all(self, model_class=None, opts=None):
        """Return all models of a model_class.

        :param model_class: the name of the model class (e.g. "Sample")
        :type model_class: basestring
        :param opts:
        :type opts:
        :return:
        :rtype:
        """
        return self.__query_helper("all", query={}, model_class=model_class, opts=opts)

    @staticmethod
    def _match_query(query, model_dict):
        """Matches a query against a model dictionary.

        :param query: query dictionary
        :type query: dict
        :param model_dict: model dictionary (e.g. model.__dict__)
        :type model_dict: dict
        :return: whether the model matches the query
        :rtype: bool
        """
        for query_key in query:
            if query_key not in model_dict:
                return False
            query_val = query[query_key]
            model_val = model_dict[query_key]
            if isinstance(query_val, list):
                if model_val not in query_val:
                    return False
            elif model_val != query_val:
                return False
        return {k: model_dict[k] for k in query}

    @classmethod
    def _find_matches(cls, query, models):
        found = []
        found_queries = []
        for m in models:
            match = cls._match_query(query, m._get_data())
            if match:
                found.append(m)
                found_queries.append(match)
        return found, found_queries

    def update_cache(
        self,
        models: List[ModelBase],
        recursive: bool = True,
        update_session: bool = False,
    ) -> Dict[str, List[ModelBase]]:
        """Update the browser's model cache from a list of models.

        .. versionchanged:: 0.1.5a8
            Added 'update_session' kwarg which will change the model's
            session to the browser's session.

        .. warning::
            If `update_session=True`, the model's session
            will change and this may have unintended consequences.

        :param models: list of models
        :param recursive: if True, recursively collect all models contained in
            the relationships and use those to update the cache as well.
        :param update_session: if True, attach the browser's session to the models
        :return: the updated dictionary broken down by model class name
        """
        assert isinstance(models, list)
        if recursive:
            memo = {}
            ModelBase._flatten_deserialized_data(models, memo)
            models = list(memo.values())
        if update_session:
            for m in models:
                m._session = self.session
        return self._group_models_and_update_cache(models)

    # TODO: do we really want to simply overwrite the dictionary or update the models?
    def _update_model_cache_helper(
        self, modelname: str, modeldict: Dict
    ) -> List[ModelBase]:
        """Updates the browser's model cache with models from the provided
        model dict."""
        self.log.info(
            "CACHE updated cached with {} {} models".format(len(modeldict), modelname)
        )
        self.model_cache.setdefault(modelname, {})

        model_cache_dict = self.model_cache[modelname]
        for mid in modeldict:
            model = modeldict[mid]
            if mid in model_cache_dict:
                cached_model = model_cache_dict[mid]
                vars(cached_model).update(vars(model))
            else:
                model_cache_dict[mid] = model
        return [model_cache_dict[mid] for mid in modeldict]

    def _group_models_and_update_cache(self, models):
        grouped_by_type = {}
        for model in models:
            classname = model.__class__.__name__
            arr = grouped_by_type.setdefault(classname, [])
            if model is not None:
                arr.append(model)

        result = {}
        for clstype in grouped_by_type:
            result[clstype] = self._update_model_cache_helper(
                clstype, {m._primary_key: m for m in grouped_by_type[clstype]}
            )
        return result

    # TODO: support unsaved models as well
    def cached_find(self, model_class, id):
        if isinstance(id, list):
            return self.cached_where({"id": id}, model_class)
        cached_models = self.model_cache.get(model_class, {})
        found_model = cached_models.get(id, None)
        if found_model is None:
            found_model = self.interface(model_class).find(id)
        else:
            self.log.info(
                "CACHE found {} model with id={} in cache".format(model_class, id)
            )
        if found_model is None:
            return None
        return self._update_model_cache_helper(
            model_class, {found_model.id: found_model}
        )[0]

    def server_where(self, query, model, opts, include, methods, page_size):
        return self.interface(model).where(
            query, opts=opts, methods=methods, include=include, page_size=page_size
        )

    def cached_where(
        self,
        query,
        model,
        primary_key="id",
        methods: Dict = None,
        include: Dict = None,
        opts: Dict = None,
    ):
        if isinstance(query, str):
            server_models = self.server_where(
                query,
                model,
                opts=opts,
                methods=methods,
                include=include,
                page_size=None,
            )
            found_dict = {}
        elif [] in query.values():
            return []
        else:
            cached_models = self.model_cache.get(model, {})
            found, found_queries = self._find_matches(
                query, list(cached_models.values())
            )
            found_dict = {f.id: f for f in found}

            # TODO: this code is broken, remaining query
            remaining_query = dict(query)
            if primary_key in query:
                found_ids = [q[primary_key] for q in found_queries]
                query_id_list = query[primary_key]
                if isinstance(query_id_list, str) or isinstance(query_id_list, int):
                    query_id_list = [query_id_list]
                remaining_ids = list(set(query_id_list).difference(set(found_ids)))
                remaining_query[primary_key] = remaining_ids
            self.log.info(
                "CACHE found {num} {model} models in cache using query {query}".format(
                    num=len(found_dict), model=model, query=self.log.pprint_data(query)
                )
            )

            # TODO: this code may be sketchy... here {'id': []}, really means we found
            #       all of the models..
            if primary_key in remaining_query and not remaining_query[primary_key]:
                return list(found_dict.values())
            server_models = self.interface(model).where(remaining_query, opts=opts)

        models_dict = OrderedDict({s.id: s for s in server_models})
        models_dict.update(found_dict)

        returned = self._update_model_cache_helper(model, models_dict)
        if opts:
            if opts.get("reverse", True) is False:
                returned = returned[::-1]
            if opts.get("limit", None):
                returned = returned[-opts["limit"] :]
        return returned

    def _search_helper(self, pattern, filter_fxn, sample_type=None, **query):
        sample_type_id = None
        if sample_type is not None:
            if isinstance(sample_type, pydent_models.SampleType):
                sample_type_id = sample_type.id
            else:
                sample_type_id = self.find_by_name(sample_type, "SampleType").id

        if sample_type_id is not None:
            query.update({"sample_type_id": sample_type_id})

        model_list = self.list_models()
        self.log.info(
            "SEARCH found {} total models of type {}".format(
                len(model_list), self.model_name
            )
        )
        matches = filter_fxn(pattern, model_list)

        if not matches:
            return []

        if query:
            query.update({"id": matches})
            filtered = self.where(query, self.model_name)
        else:
            filtered = self.interface().find(matches)
        self.log.info(
            "SEARCH filtered to {} total models of type {}".format(
                len(filtered), self.model_name
            )
        )

        if self.use_cache:
            self.update_cache(filtered)
        return filtered

    def search(self, pattern, ignore_case=True, sample_type=None, **query):
        """Performs a regular expression search of Samples.

        :param pattern: regular expression pattern
        :type pattern: basestring
        :param ignore_case: whether to ignore case for regex search (default: True)
        :type ignore_case: bool
        :param sample_type: sample_type_name to filter samples (optional)
        :type sample_type: basestring
        :param query: additional query parameters to filter by
        :type query: dict
        :return: list of samples
        :rtype: list
        """

        def regex_filter(pattern, sample_list):
            matches = []
            for name in sample_list:
                args = []
                if ignore_case:
                    args.append(re.IGNORECASE)
                m = re.search(pattern, name, *args)
                if m is not None:
                    matches.append(name)
            return matches

        return self._search_helper(
            pattern, regex_filter, sample_type=sample_type, **query
        )

    def search_description(
        self, pattern, samples=None, sample_type=None, ignore_case=True
    ):
        """Search samples by their description.

        :param pattern: regex pattern
        :type pattern: basestring
        :param samples: samples to search. If left blank, a search to find samples will
            be performed
        :type samples: list
        :param sample_type: restrict to a particular sample type
        :type sample_type: name
        :param ignore_case: ignore case for regular pattern search (default True)
        :type ignore_case: bool
        :return: list of samples
        :rtype: list
        """
        matches = []
        args = []
        if samples is None:
            samples = self.search(".*", sample_type=sample_type)
        if ignore_case:
            args.append(re.IGNORECASE)
        for sample in samples:
            description = sample.description
            if description is None:
                continue
            if re.search(pattern, sample.description, *args):
                matches.append(sample)
        return matches

    def close_matches(self, pattern, sample_type=None, **query):
        """Finds samples whose names closely match the pattern.

        :param pattern: regular expression pattern
        :type pattern: basestring
        :param sample_type: sample_type_name to filter samples (optional)
        :type sample_type: basestring
        :param query: additional query parameters to filter by
        :type query: dict
        :return: list of samples
        :rtype: list
        """
        return self._search_helper(
            pattern, get_close_matches, sample_type=sample_type, **query
        )

    def list_field_values(self, model_ids, **query):
        """Lists sample field values.

        May supply an additional query to filter :class:`FieldValue`.
        """
        query.update({"parent_class": self.model_name, "parent_id": model_ids})
        return self.where(query, "FieldValue")

    @staticmethod
    def _group_by_attribute(models, attribute):
        """Group models by the given attribute."""
        d = {}
        for s in models:
            arr = d.setdefault(getattr(s, attribute), [])
            arr.append(s)
        return d

    def new_sample(self, sample_type, name, description, project, properties=None):
        st = self.find_by_name(sample_type, "SampleType", primary_key="name")
        if st._get_deserialized_data().get("field_types", None) is None:
            fts = self.where(
                {"parent_class": "SampleType", "parent_id": st.id}, "FieldType"
            )
            st.field_types = fts
        return st.new_sample(name, description, project, properties=properties)

    # TODO: handle not-yet existant samples using rid
    def _retrieve_has_many_or_has_one(
        self, models, relationship_name, relation=None, strict=True
    ):
        """Performs exactly 1 query to fullfill some relationship for a list of
        models."""
        if not models:
            return []
        models = models[:]
        if relation is None:
            relation = models[0].get_relationships()[relationship_name]

        ref = relation.ref  # sample_id
        attr = relation.attr  # id
        model_class2 = relation.nested

        # todo: collect existing fullfilled relationship
        # todo: partition, then collect callback
        # todo: how to handle when model_attr is absent?, or just raise error?

        retrieve_query = relation.build_query(models)
        retrieved_models = self.where(retrieve_query, model_class2)
        self.log.info(
            "RETRIEVE retrieved {num} {cls} models using query {query}".format(
                num=len(retrieved_models),
                cls=model_class2,
                query=self.log.pprint_data(retrieve_query),
            )
        )

        if not retrieved_models:
            return []

        if relation.QUERY_TYPE == "query":
            model_dict = {m1.id: list() for m1 in models}
            for model in retrieved_models:
                model_ref = getattr(model, ref)
                if model_ref is not None:
                    if not strict:
                        model_dict.setdefault(model_ref, []).append(model)
                    else:
                        model_dict[model_ref].append(model)
                else:
                    self.log.error(
                        "RETRIEVE ref: {ref} {model_ref}, attr: {attr}".format(
                            ref=ref, attr=attr, model_ref=model_ref
                        )
                    )
        elif relation.QUERY_TYPE == "by_id":
            retrieved_dict = {getattr(m2, attr): m2 for m2 in retrieved_models}
            model_dict = {m1.id: None for m1 in models}
            missing_models = []
            for model in models:
                model_attr = getattr(model, attr)
                model_ref = getattr(model, ref)
                if model_ref is not None:
                    if model_attr is not None:
                        if model_ref not in retrieved_dict:
                            self.log.info(strict)
                        else:
                            model_dict[model_attr] = retrieved_dict[model_ref]
                    else:
                        self.log.error(
                            "attr: {attr}={model_attr}, ref: {ref}={model_ref}".format(
                                ref=ref,
                                attr=attr,
                                model_attr=model_attr,
                                model_ref=model_ref,
                            )
                        )
                    if model_ref not in retrieved_dict:
                        missing_models.append(model_ref)
            if missing_models:
                self.log.error(
                    "INCONSISTENT AQUARIUM DATABASE - There where {models} missing {cls} "
                    "models "
                    "from the Aquarium database, which were ignored by trident. "
                    "This happens when models are deleted from Aquarium which "
                    "results in an inconsistent server database. Trident was unable "
                    "resolve the following relationships which returned no models "
                    "from the server: "
                    "{cls}.where({attr}={missing})".format(
                        models=len(missing_models),
                        cls=model_class2,
                        missing=missing_models,
                        attr=attr,
                    )
                )
        else:
            raise BrowserException(
                "QUERY_TYPE '{}' for relation '{}' not recognized.".format(
                    relation.QUERY_TYPE, relationship_name
                )
            )
        for model in models:
            found_models = model_dict[getattr(model, attr)]
            setattr(model, relationship_name, found_models)

        return retrieved_models

    def _retrieve_has_many_through(
        self, models: List[ModelBase], relationship_name: str, strict: bool = True
    ):
        """Performs exactly 2 queries to establish a HasManyThrough
        relationship."""
        relation = models[0].get_relationships()[relationship_name]
        association_relation = models[0].get_relationships()[
            relation.through_model_attr
        ]
        attr = relation.attr
        ref = association_relation.ref

        # find other key
        association_class = ModelRegistry.get_model(association_relation.nested)
        association_relationships = association_class.get_relationships()
        other_ref = None
        for r in association_relationships:
            ar = association_relationships[r]
            if ar.nested == relation.nested:
                other_ref = r

        associations = self._retrieve_has_many_or_has_one(
            models, relation.through_model_attr, strict=strict
        )
        self._retrieve_has_many_or_has_one(associations, other_ref, strict=strict)

        associations_by_mid = {}
        for a in associations:
            associations_by_mid.setdefault(getattr(a, ref), []).append(a)

        all_models = []
        for m in models:
            mid = getattr(m, attr)
            if mid in associations_by_mid:
                associations = associations_by_mid[mid]
                _models = [getattr(a, other_ref) for a in associations]
                setattr(m, relationship_name, _models)
                all_models += _models
            else:
                setattr(m, relationship_name, None)
        return list(set(all_models))

    @staticmethod
    def _get_relation_from_model(model, relationship_name, strict):
        relationships = model.get_relationships()
        relation = relationships.get(relationship_name, None)
        if relation is None:
            if strict:
                msg = did_you_mean(relationship_name, list(relationships))
                if not msg:
                    msg = "Available relations: {}".format(pformat(list(relationships)))
                raise BrowserException(
                    "Relation '{}' not found in relationships for {}.\nHint: {}".format(
                        relationship_name, type(model), msg
                    )
                )
        return relation

    def retrieve(
        self,
        models: List[ModelBase],
        relationship_name: str,
        relation: BaseRelationship = None,
        strict: bool = True,
        force_refresh: bool = False,
    ) -> List[ModelBase]:
        """Retrieves a model relationship for the list of models. Compared to a
        `for` loop, `retrieve` is >10X faster for most queries.

        .. code-block:: python

            # FAST code example
            samples = browser.search(".*mCherry.*")
            items = browser.retrieve(samples, 'items')
            for s in samples:
                sample_items = s.items
                # do something with items

        .. code-block:: python

            # very SLOW code example. DO NOT USE.
            samples = browser.search(".*mCherry.*")
            for s in samples:
                sample_items = s.items
                # do something with itmes

        :param models: list of models to retrieve the attribute
        :type models: list
        :param relationship_name: name of the attribute to retrieve
        :type relationship_name: basestring
        :param relation: the relation to retrieve (operational)
        :type relation: pydent.relationships.Relation
        :param strict: wither to ignore database inconsistencies
        :type strict: bool
        :return: list of models retrieved
        :rtype: list
        """
        if not models:
            return []
        self.log.info('RETRIEVE retrieving "{}"'.format(relationship_name))
        model_classes = {m.__class__.__name__ for m in models}
        assert (
            len(model_classes) == 1
        ), "Models must be all of the same BaseModel, but found {}".format(
            model_classes
        )
        if relation is None:
            relation = self._get_relation_from_model(
                models[0], relationship_name, strict
            )
            if relation is None:
                return []
        else:
            if relationship_name in models[0].get_relationships():
                raise BrowserException(
                    'Cannot add new relationship "{}" because it already exists in the '
                    "model definition".format(relationship_name)
                )
        # TODO: add relationship handler for Many and One
        if relation.__class__.__name__ not in self.ACCEPTED_GET_RELATION_TYPES:
            raise BrowserException(
                'retrieve is not supported for the "{}" relationship'.format(
                    relation.__class__.__name__
                )
            )
        self.log.info("RETRIEVE {}: {}".format(relationship_name, relation))

        if not force_refresh:
            needs_refresh = [
                m for m in models if not m.is_deserialized(relationship_name)
            ]
            no_refresh = [m for m in models if m.is_deserialized(relationship_name)]
        else:
            needs_refresh = models
            no_refresh = []

        if needs_refresh:
            if hasattr(relation, "through_model_attr"):
                found_models = self._retrieve_has_many_through(
                    needs_refresh, relationship_name, strict=strict
                )
            else:
                found_models = self._retrieve_has_many_or_has_one(
                    needs_refresh, relationship_name, relation, strict=strict
                )
        else:
            found_models = []

        self.log.info(
            'RETRIEVE retrieved {} for "{}"'.format(
                len(found_models), relationship_name
            )
        )
        for model in no_refresh:
            val = getattr(model, relationship_name)
            if isinstance(val, list):
                found_models += val
            else:
                found_models.append(val)
        return list(set(found_models))

    def recursive_retrieve(
        self,
        models: List[ModelBase],
        relations: Union[str, List[BaseRelationship], Dict],
        strict: bool = True,
        force_refresh: bool = False,
    ):
        """Efficiently retrieve a model relationship recursively from an
        iterable. The relations_dict iterable may be either a list or a
        dictionary. For example, the following will collect all of the
        field_values and their incoming and outgoing wires, the connecting
        field_values, and finally those FieldValues' operations.

        .. code-block::

            relation_dict = {
                "field_values": {
                    "wires_as_dest": {
                        "source": "operation",
                        "destination": "operation"
                    },
                    "wires_as_source": {
                        "source": "operation",
                        "destination": "operation"
                    },
                }
            }
            browser.retrieve(operations, relation_dict)

        :param models: models to retrieve from
        :type models: list
        :param relations: the relation to retrieve. May be a string (by \
            attribute name), a list, or a dict.
        :type relations: list|dict|basestring
        :param strict: wither to ignore database inconsistencies
        :param force_refresh:
        :type force_refresh: bool
        :return: dictionary of all models retrieved grouped by the attribute \
            name that retrieved them.
        :rtype: dictionary
        """
        self.log.info("RETRIEVE recursively retrieving {}".format(relations))
        if isinstance(relations, str):
            self.log.info('RETRIEVE retrieving "{}"'.format(relations))
            return {
                relations: self.retrieve(
                    models, relations, strict=strict, force_refresh=force_refresh
                )
            }
        elif (
            isinstance(relations, list)
            or isinstance(relations, set)
            or isinstance(relations, dict)
            or isinstance(relations, tuple)
        ):
            models_by_attr = {}
            for relation_name in relations:
                models_by_attr.setdefault(relation_name, [])
                new_models = self.retrieve(
                    models, relation_name, strict=strict, force_refresh=force_refresh
                )
                models_by_attr[relation_name] += new_models
                if isinstance(relations, dict):
                    _models_by_attr = self.recursive_retrieve(
                        new_models,
                        dict(relations).pop(relation_name),
                        strict=strict,
                        force_refresh=force_refresh,
                    )
                    for attr in _models_by_attr:
                        _models = _models_by_attr[attr]
                        if attr in models_by_attr:
                            models_by_attr[attr] += _models
                        else:
                            models_by_attr[attr] = _models_by_attr[attr]
            return models_by_attr
        elif not strict:
            return []
        else:
            raise BrowserException(
                "Type {} for is not recognized for recursive_retrieve".format(
                    type(relations)
                )
            )

    @classmethod
    def sample_network(
        cls,
        samples: List[Sample],
        reverse: bool = False,
        g: nx.DiGraph = None,
        get_models: Callable = None,
        cache_func: Callable = None,
        key_func: Callable = None,
    ) -> nx.DiGraph:
        """Build a DAG of :class:`Samples <pydent.models.Sample>` from their.

        :class:`FieldValues <pydent.models.FieldValue>`.

        .. versionadded:: 0.1.5a7
            method added

        .. versionchanged:: 0.1.5a16
            added optional get_models, cache_func, key_func arguments

        :param samples: list of samples
        :param reverse: whether to reverse the edges of the final graph
        :param g: the graph
        :return:
        """

        if not get_models:

            def get_models(model):
                for fv in model.field_values:
                    if fv.sample and issubclass(type(fv.sample), ModelBase):
                        yield fv.sample, {"field_value_name": fv.name}

        if not cache_func:

            def cache_func(models):
                sess = models[0].session
                sess.browser.get(models, {"field_values": "sample"})

        return cls.relationship_network(
            samples,
            get_models=get_models,
            cache_func=cache_func,
            reverse=reverse,
            g=g,
            key_func=key_func,
        )

    @classmethod
    def relationship_network(
        cls,
        models: List[ModelBase],
        get_models: Callable,
        cache_func: Callable = None,
        key_func: Callable = None,
        reverse: bool = False,
        g: nx.DiGraph = None,
        strict_cache: bool = True,
    ):
        """Build a DAG of related models based on some relationships. By
        default are built from a model using (model.__class__.__name__,
        model._primary_key)

        .. versionadded:: 0.1.5a7
            method added

        .. seealso::
            Usage example :meth:`sample_network <pydent.browser.Browser.sample_network>`

        :param models: list of models
        :param get_models: A function that takes in a single instance of a Model
            and returns an iterable of Tuples of (model, data),
            which is used to build edges.
        :param key_func: An optional function that takes in a single instance of
            a Model and returns a key and some node data
        :param cache_func: A function that takes in a list of models and caches
            some results.
        :param g: optional nx.DiGraph
        :param reverse: whether to reverse the edge list
        :param strict_cache: if True, if a request occurs after the cache step,
            a ForbiddenRequestException will be raised.
        :return: the relationship graph
        """

        if key_func is None:

            def key_func(m):
                return (m.__class__.__name__, m._primary_key), {}

        if g is None:
            g = nx.DiGraph()

        models = list(models)

        if not models:
            return g

        for m in models:
            key, ndata = key_func(m)
            g.add_node(key, attr_dict=ndata)

        visited = list(g.nodes)

        if cache_func:
            cache_func(models)

        new_models = []
        new_edges = []

        if strict_cache:
            kwargs = {"using_requests": False, "session_swap": True}
        else:
            kwargs = {}

        with models[0].session(**kwargs):
            try:
                for m1 in models:
                    for m2, data in get_models(m1):
                        if key_func(m2)[0] not in visited:
                            new_models.append(m2)
                        new_edges.append((key_func(m2)[0], key_func(m1)[0], data))
            except ForbiddenRequestError as e:
                msg = (
                    "An exception occurred while strict_cache == True.\n"
                    "This is most likely due to the cache_func not being thorough.\n"
                    "{}".format(str(e))
                )
                raise e.__class__(msg)

        for n1, n2, edata in new_edges:
            if reverse:
                g.add_edge(n2, n1, attr_dict=edata)
            else:
                g.add_edge(n1, n2, attr_dict=edata)

        return cls.relationship_network(
            new_models,
            get_models,
            cache_func,
            key_func=key_func,
            g=g,
            reverse=reverse,
            strict_cache=strict_cache,
        )

    # def relationship_network(self, models, get_models, cache_func, g=None):
    #     """
    #
    #
    #     .. versionadded:: 0.1.5a7
    #
    #
    #     :param models:
    #     :param get_models:
    #     :param g:
    #     :return:
    #     """
    #
    #     def model_to_key(m):
    #         return (m.__class__.__name__, m.id)
    #
    #     if g is None:
    #         g = nx.DiGraph()
    #     for m in models:
    #         g.add_node(model_to_key(m))
    #
    #     cache_func(models)
    #
    #     for m in models:
    #         next_models = get_models(m)

    def get(
        self,
        models: List[ModelBase],
        relations: List[BaseRelationship] = None,
        query: dict = None,
        strict: bool = True,
        force_refresh: bool = False,
    ) -> Union[Dict[str, List[ModelBase]], List[ModelBase]]:
        """

        :param models:
        :param relations:
        :param query:
        :param strict:
        :param force_refresh:
        :return:
        """
        if isinstance(models, ModelBase):
            models = [models]
        elif isinstance(models, str):
            models = list(self.model_cache.get(models, {}).values())
            if query:
                models, _ = self._find_matches(query, models)
        if relations:
            if isinstance(relations, str):
                return self.retrieve(
                    models, relations, strict=strict, force_refresh=force_refresh
                )
            else:
                return self.recursive_retrieve(
                    models, relations, strict=strict, force_refresh=force_refresh
                )
        else:
            return models

    def export_samples_to_csv(self, samples, out):
        """Exports the samples to a csv (for Aquarium import)

        :param samples: list of samples
        :type samples: list
        :param out: output path of csv file
        :type out: basestring
        :return: pandas dataframe used for the csv
        :rtype: pandas.DataFrame
        """
        df = self.samples_to_df(samples)
        df.to_csv(out)
        return df

    def samples_to_rows(self, samples, sample_resolver=None):
        """Return row of dictionaries containing sample information and their
        properties. Can be imported into a pandas DataFrame:

        .. code-block:: python

            import pandas

            df = pandas.DataFrame(samples_to_rows(samples))

        :param samples: samples, all of same sample type
        :type samples: list
        :param sample_resolver: callable to resolve a sample object if the \
            field value property contains a sample. \
            Defaults to return the sample name.
        :type sample_resolver: callable
        :return: list of dictionaries containing sample info and properties
        :rtype: list
        """

        assert (
            len({s.sample_type_id for s in samples}) == 1
        ), "Samples be the of the same SampleType"
        sample_type = samples[0].sample_type

        # self.recursive_retrieve(samples, {
        #     "field_values": "sample",
        #     "sample_type": {
        #         "field_types"
        #     }
        # })
        if sample_resolver is None:

            def default_resolver(sample):
                if isinstance(sample, pydent_models.Sample):
                    return sample.name
                if isinstance(sample, list):
                    resolved_samples = []
                    for s in sample:
                        if isinstance(s, pydent_models.Sample):
                            resolved_samples.append(s.name)
                        else:
                            resolved_samples.append(s)
                    sample = resolved_samples
                return sample

            sample_resolver = default_resolver
        rows = []
        for s in samples:
            row = {
                sample_type.name: s.name,
                "Description": s.description,
                "Project": s.project,
            }
            props = {k: sample_resolver(v) for k, v in s.properties.items()}
            row.update(props)
            rows.append(row)
        return rows

    # TODO: add to change log
    @staticmethod
    def _inspect_header(model, name=None, _id="id"):
        if name is not None:
            name = eval("model.{}".format(name))
        else:
            name = ""
        return "{model_class}: {_id} {name}".format(
            model_class=model.__class__.__name__,
            name=name,
            _id=eval("model.{}".format(_id)),
        )

    # TODO: add to change log
    @staticmethod
    def _inspect_rows(model, attributes, indent=2):
        rows = []
        for attr in attributes:
            rows.append(
                "{attr}: {val}".format(attr=attr, val=eval("model.{}".format(attr)))
            )
        sep = "\n" + " " * indent
        s = sep + sep.join(rows)
        return s

    # TODO: add to change log
    @classmethod
    def inspect(cls, model, header, attributes):
        model_header = cls._inspect_header(model, **header)
        rows = cls._inspect_rows(model, attributes)
        return "{header}{rows}".format(header=model_header, rows=rows)


import sys

import torch
import requests

from PIL import Image
from torchvision import transforms
from matplotlib import pyplot as plt

# from models.clipseg import ClipDensePredT
from models.clipseg import CLIPDensePredT

# load model
# model = ClipDensePredT(version="ViT-B/16", reduce_dim=64)
model = CLIPDensePredT(version="ViT-B/16", reduce_dim=64)
model.eval()

model.load_state_dict(torch.load("weights/rd64-uni.pth", map_location=torch.device('cpu')), strict=False)

# load image
input_image = Image.open("example_image.jpg")
# input_image = Image.open("tree.jpg")
# input_image = Image.open("Unknown.png")
prompt_image = Image.open("wood.jpg")
# prompt_image = Image.open("Unknown-2.png")
# prompt_image = Image.open("spoon.jpg")

# normalize image
transform = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize(mean=[.485, .456, .406], std=[.229, .224, .225]),
    transforms.Resize((352, 352))
    # transforms.Resize((160, 160))
])
img = transform(input_image).unsqueeze(0)
pimg = transform(prompt_image).unsqueeze(0)
print(prompt_image.size)
print(pimg.shape)

# set prompt
prompts = ["something to fill"]
# prompts = sys.argv[1:]

import time
s = time.time()

# # prediction
with torch.no_grad():
    pred = model(img.repeat(1, 1, 1, 1), prompts)[0]
    # pred = model(img, pimg)[0]

e = time.time()

print(e-s)

# visualize prediction
_, axs = plt.subplots(1, 2, figsize=(9, 4))
for i, ax in enumerate(axs.flatten()):
    ax.axis("off")
    if i == 0:
        ax.text(0, -15, "input image")
        ax.imshow(input_image)
    else:
        masked = torch.sigmoid(pred[i-1][0])
        ax.text(0, -15, prompts[i-1])
        ax.imshow(masked)

# pred = torch.sigmoid(pred)
# print(pred.shape)
# print(pred.dtype, pred.min(), pred.max())

# plt.imshow(input_image)
# plt.imsave("out.png", torch.sigmoid(pred[0][0]))

_.savefig("out.png")


from typing import List

"""
Summary: Iterate over the array in a while loop for len(arr) times. Pointer
starts at 0. If element is not zero, move pointer -> one step. Else, pop the
item and append to the end, don't move the pointer, the list shifts left.
------------------------------------------------------------------------------

https://leetcode.com/problems/move-zeroes/

Given an integer array nums, move all 0's to the end of it while maintaining
the relative order of the non-zero elements.

Note that you must do this in-place without making a copy of the array.
"""

class Solution:

    # T: at least O(N2); S: O(1), is it though?
    def moveZeroes(self, nums: List[int]) -> None:
        """
        Do not return anything, modify nums in-place instead.
        """
        length = len(nums)
        iteration = 0
        pointer = 0
        while iteration < length:
            if nums[pointer] == 0:
                nums.append(nums.pop(pointer))
            else:
                pointer += 1
            iteration += 1

    # This one further optimises you top one - you keep iterating over 0s as
    # well, which is not necessary.
    def moveZeroes(self, nums):
        i = 0
        end = len(nums)
        while i < end:
            if nums[i] == 0:
                del nums[i]
                nums.append(0)
                end -= 1
            else:
                i += 1

    # Interesting solution
    def moveZeroes(self, nums: List[int]) -> None:
        count = 0
        n = len(nums)
        for i in range(n):
            if nums[i] == 0:
                count += 1
            elif count != 0:
                nums[i - count] = nums[i]
                nums[i] = 0

    # Cheesy one haha
    def moveZeroes(self, nums: List[int]) -> None:
        nums[:] = [num for num in nums if num] + [0] * nums.count(0)

def main():
    nums = [0, 1, 0, 3, 12]
    Solution().moveZeroes(nums)
    print(nums)

if __name__ == "__main__":
    main()


# USAGE
# python opencv-crop.py

# loading image from disk loads as NumPy array
# restate image = NumPy array
# to crop you need to know how opencv slices
# arrays start at 0 so the numbers are not inclusive
# ratio is row:column format
# if you go :, 3 you won't get 3..youll get up to 2
#  0  1 | 2|  3  4  5
# 10 11 |12| 13 14 15
# 20 21 |22| 23 24 25
# a[:,2] would grab the column 2, 12, 22

# NumPy is a vectorized library that is very fast

# import necessary packages
import argparse
import cv2

# construct arg parser
ap = argparse.ArgumentParser()
ap.add_argument("-i","--image", type=str, default="avery.jpg",
                help="path to input image")
args = vars(ap.parse_args())

# load explanation image
# python
image = cv2.imread(args["image"])
cv2.imshow("Original", image)
cv2.waitKey(0)

# face = image
# ROI = region of interest is in Y (rows),X (columns) = array
# startY:endY
# startX:endX
# inverse from X,Y coordinates
head = image[766:886,1114:1226]
cv2.imshow("Head", head)
cv2.waitKey(0)

body = image[736:900,1226:1656]
cv2.imshow("Body",body)
cv2.waitKey(0)


"""Unit tests model_utils functions."""
from remedi import models

import pytest
import torch

def assert_equals(actual, expected, path="actual"):
    """Simple implementation of torch-friendly deep equality."""
    assert type(actual) is type(expected), path
    if isinstance(actual, torch.Tensor):
        assert actual.float().allclose(expected.float()), path
    elif isinstance(actual, dict):
        for key, value in actual.items():
            child_path = f'{path}["{key}"]'
            assert key in expected, child_path
            assert_equals(value, expected[key], path=child_path)
    elif isinstance(actual, (list, tuple)):
        for index, value in enumerate(actual):
            child_path = f"{path}[{index}]"
            assert_equals(value, expected[index], path=child_path)

@pytest.mark.parametrize(
    "value,device,dtype",
    (
        ("x", None, None),
        (torch.tensor([1, 2, 3]), None, None),
        (torch.tensor([1, 2, 3]), None, torch.float),
        ({"foo": "bar"}, "cpu", None),
        ([torch.Tensor([1, 2, 3])], "cpu", None),
        ({"foo": torch.tensor([1, 2, 3])}, "cpu", None),
        ({"foo": [torch.tensor([1, 2, 3])]}, "cpu", None),
        ({"foo": [torch.tensor([1, 2, 3])]}, "cpu", torch.float),
    ),
)
def test_map_to_preserves_values(value, device, dtype):
    """Test map_location returns correct value."""
    actual = models.map_to(value, device=device, dtype=dtype)
    assert_equals(actual, value)


import cv2
import numpy as np

def find_list_files(pattern_filename, path):
    '''
    Find list of files following a pattern filename within a path
    This returns a list of results, void list [] if items not found
    '''
    import os,fnmatch
    result = []
    for root, dirs, files in os.walk(path): # os walk is a generator
        for name in files:
            if fnmatch.fnmatch(name, pattern_filename):
                result.append(os.path.join(root, name))
    return result

# Goal: To build numpy array (num_samples,win_edge1,win_edge2)

class LandsatDataLoader:
    def __init__(self, input_folder):
        self.X_train = None
        self.input_files  = [f for f in find_list_files("LS*png",input_folder)]
        print("Number of input files loaded:",len(self.input_files))

    def _load_image(self,ifile):
        image = cv2.imread(ifile)
        return cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

    def load_data(self, randomization=False):
        #num_samples = len(input_files)
        #print(len(self.input_files))
        list_samples = []

        for ifile in self.input_files:
            list_samples.append(self._load_image(ifile))

        if randomization:
            seed=42
            print("Applying Randomization on list of input samples seed {}".format(seed))
            import random
            random.seed(seed)
            random.shuffle(list_samples)

        X_train = np.array(list_samples)
        self.X_train = X_train
        return X_train

    def load_data_and_filenames(self):
        filenames = []
        list_samples = []
        for ifile in self.input_files:
            list_samples.append(self._load_image(ifile))
            filenames.append(ifile)

        X_train = np.array(list_samples)
        self.X_train = X_train
        return X_train, filenames

    def normal_sample(self):
        pass

    def abnormal_sample(self):
        pass

if __name__ == '__main__':
    input_folder = '/QCOLT/QCOLT_DEV_OPS//TDS_NOVELTY_DETECTION/EXP_02//nominal_chips/'
    dataLoader = LandsatDataLoader(input_folder)
    data = dataLoader.load_data()
    print(data.shape)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.http import HttpResponse
from .models import Tag, User, Image
from rest_framework import status
from rest_framework.request import Request
import json
from django.views.decorators.http import require_http_methods
import logging
from .utils import validate_token, get_download_url

@require_http_methods(['POST', 'GET'])
def create_image(request: Request) -> HttpResponse:
    try:
        logging.error(request)
        logging.error(request.body)
        try:
            token_id = request.META['HTTP_AUTHORIZATION']
            user_id = validate_token(token_id)
        except Exception as exp:
            logging.error(exp)
            return HttpResponse(status=status.HTTP_401_UNAUTHORIZED)
        if request.method == 'POST':
            logging.error(request)
            if user_id is not None:
                image_data = json.loads(request.body)
                url = get_download_url(image_data['uri'], image_data['name'])
                if url == "":
                    return HttpResponse(status=status.HTTP_400_BAD_REQUEST)
                image_content = {}
                if 'timestamp' in image_data:
                    image_content['timestamp'] = image_data['timestamp']
                image_content['name'] = image_data['name']
                image_content['place'] = image_data['place']
                image_content['uri'] = url
                image_content['user_id'] = user_id
                tag_ids = []
                tags = []
                if 'tags' in image_data:
                    for tag_id in image_data['tags']:
                        tag_ids.append(tag_id)
                        tags.append(Tag.objects.get(id=int(tag_id)))
                image_instance = Image(name=image_content['name'], place=image_content['place'],
                                       uri=image_content['uri'],
                                       user_id=image_content['user_id'], identity_group=0)
                image_instance.save()
                image_instance.identity_group = image_instance.id
                image_instance.save()
                image_instance = Image.objects.get(id=image_instance.id)
                for tag in tags:
                    image_instance.tags.add(tag)
                image_instance.save()
                shared_with = [user_instance.id for user_instance in image_instance.shared_with.all()]
                image_instance_json = json.dumps({'id': image_instance.id, 'name': image_instance.name,
                                                  'place': image_instance.place, 'uri': image_instance.uri,
                                                  'timestamp': str(image_instance.timestamp),
                                                  'user_id': user_id, 'tags': tag_ids, 'shared_with': shared_with})
                return HttpResponse(image_instance_json, status=status.HTTP_201_CREATED)
            else:
                return HttpResponse(status=status.HTTP_401_UNAUTHORIZED)
        if request.method == 'GET':
            if user_id is not None:
                logging.error(request)
                own_by_me = Image.objects.filter(user_id=user_id)
                shared_with_me = Image.objects.filter(shared_with__id=user_id)
                all_images = []
                for image in own_by_me:
                    all_images.append(image.id)
                for image in shared_with_me:
                    all_images.append(image.id)
                return HttpResponse(json.dumps({"image_ids": all_images}), status=status.HTTP_200_OK)
            return HttpResponse(status=status.HTTP_401_UNAUTHORIZED)
    except Exception as ex:
        logging.error('Final Catch se ye error aayi hai ' + str(ex))
        return HttpResponse(status=status.HTTP_400_BAD_REQUEST)
    return HttpResponse(status=status.HTTP_404_NOT_FOUND)

@require_http_methods(['GET', 'PUT', 'DELETE'])
def image_crud(request: Request, image_id: int):
    logging.error(request)
    try:
        token_id = request.META['HTTP_AUTHORIZATION']
        user_id = validate_token(token_id)
    except Exception as exp:
        return HttpResponse(status=status.HTTP_401_UNAUTHORIZED)
    if user_id is not None:
        if request.method == 'GET':
            try:
                image_instance = Image.objects.get(id=image_id)
            except Exception as exp:
                return HttpResponse(status=status.HTTP_404_NOT_FOUND)
            if image_instance.user_id == user_id:
                image_instance_json = json.dumps({
                    'id': image_instance.id,
                    'name': image_instance.name,
                    'place': image_instance.place,
                    'uri': image_instance.uri,
                    'timestamp': str(image_instance.timestamp),
                    'tags': [tag_instance.id for tag_instance in image_instance.tags.all()],
                    'shared_with': [user_instance.id for user_instance in image_instance.shared_with.all()]
                })
                return HttpResponse(image_instance_json, status=status.HTTP_200_OK)
            return HttpResponse('get call kiya!!', status=status.HTTP_401_UNAUTHORIZED)
        if request.method == 'PUT':
            image_instance = Image.objects.get(id=image_id)
            if image_instance.user_id == user_id:
                query = json.loads(request.body)
                if 'name' in query:
                    image_instance.name = query['name']
                if 'place' in query:
                    image_instance.place = query['place']
                if 'uri' in query:
                    image_instance.uri = query['uri']
                if 'timestamp' in query:
                    image_instance.timestamp = query['timestamp']
                image_instance.save()
                tag_ids = []
                if 'tags' in query:
                    image_instance.tags.clear()
                    for tag_id in query['tags']:
                        tag_ids.append(tag_id)
                        image_instance.tags.add(Tag.objects.get(id=int(tag_id)))
                image_instance.save()
                image_instance_json = json.dumps({
                    'id': image_instance.id,
                    'name': image_instance.name,
                    'place': image_instance.place,
                    'uri': image_instance.uri,
                    'timestamp': str(image_instance.timestamp),
                    'tags': [tag_instance.id for tag_instance in image_instance.tags.all()]
                })
                return HttpResponse(image_instance_json, status=status.HTTP_200_OK)
        if request.method == 'DELETE':
            image_instance = Image.objects.get(id=image_id)
            if image_instance.user_id == user_id:
                image_instance.delete()
                return HttpResponse(status=status.HTTP_204_NO_CONTENT)
    return HttpResponse(status=status.HTTP_404_NOT_FOUND)

@require_http_methods(['GET'])
def get_private_images(request: Request):
    try:
        logging.error(request)
        try:
            token_id = request.META['HTTP_AUTHORIZATION']
            user_id = validate_token(token_id)
        except Exception as exp:
            return HttpResponse(status=status.HTTP_401_UNAUTHORIZED)
        if user_id is not None:
            image_instances = Image.objects.all().filter(user_id=user_id)
            image_ids = []
            for image_instance in image_instances:
                image_ids.append(image_instance.id)
            return HttpResponse(json.dumps({'image_ids': image_ids}), status=status.HTTP_200_OK)
        else:
            return HttpResponse(status=status.HTTP_401_UNAUTHORIZED)
    except Exception as exp:
        logging.error(str(exp))
        return HttpResponse(status=status.HTTP_404_NOT_FOUND)

@require_http_methods(['GET'])
def get_shared_images(request):
    try:
        logging.error(request)
        token_id = request.META['HTTP_AUTHORIZATION']
        user_id = validate_token(token_id)
        if user_id is not None:
            image_instances = Image.objects.filter(shared_with__id=user_id)
            image_ids = []
            for image_instance in image_instances:
                image_ids.append(image_instance.id)
            return HttpResponse(json.dumps({"image_ids": image_ids}), status=status.HTTP_200_OK)
        return HttpResponse(status=status.HTTP_401_UNAUTHORIZED)
    except Exception as exp:
        logging.error("exception has occured: " + str(exp))
        return HttpResponse(status=status.HTTP_404_NOT_FOUND)

def image_share(request: Request, image_id: int):
    try:
        logging.error(request)
        try:
            token_id = request.META['HTTP_AUTHORIZATION']
            user_id = validate_token(token_id)
            user_ids = json.loads(request.body)['user_ids']
        except Exception as exp:
            return HttpResponse(status=status.HTTP_401_UNAUTHORIZED)
        if user_id is not None:
            image_instance = Image.objects.get(id=image_id)
            if image_instance.user_id == user_id:
                try:
                    if len(user_ids) != len(set(user_ids)) or len(user_ids) == 0:
                        return HttpResponse(status=status.HTTP_400_BAD_REQUEST)
                    for user_id_ in user_ids:
                        if user_id == user_id_:
                            return HttpResponse(status=status.HTTP_403_FORBIDDEN)
                        image_instance.shared_with.add(User.objects.get(id=int(user_id_)))
                except Exception as exp:
                    return HttpResponse(status=status.HTTP_400_BAD_REQUEST)
                image_instance.save()
                return HttpResponse(status=status.HTTP_204_NO_CONTENT)
            return HttpResponse(status=status.HTTP_404_NOT_FOUND)
    except Exception as ex:
        return HttpResponse(status=status.HTTP_400_BAD_REQUEST)

    return HttpResponse(status=status.HTTP_404_NOT_FOUND)

@require_http_methods(['GET', 'POST'])
def identical_image(request, image_id: int):
    try:
        logging.error(request)
        token_id = request.META['HTTP_AUTHORIZATION']
        user_id = validate_token(token_id)
    except Exception as exp:
        return HttpResponse(status=status.HTTP_401_UNAUTHORIZED)
    if user_id is not None:
        try:
            image_instance = Image.objects.get(id=image_id)
        except Exception as exp:
            return HttpResponse(status=status.HTTP_404_NOT_FOUND)
        if user_id == image_instance.user_id:
            if request.method == 'GET':
                identity_group = image_instance.identity_group
                identical_images = Image.objects.filter(identity_group=identity_group)
                image_ids = []
                for image in identical_images:
                    if int(image.id) != int(image_id):
                        image_ids.append(image.id)
                return HttpResponse(json.dumps({"image_ids": image_ids}), status=status.HTTP_200_OK)

            if request.method == 'POST':
                identity_group = image_instance.identity_group
                logging.error(request.body)
                image_ids = json.loads(request.body)['image_ids']
                if len(image_ids) != len(set(image_ids)) or len(image_ids) == 0:
                    return HttpResponse(status=status.HTTP_400_BAD_REQUEST)
                try:
                    identical_images = []
                    for image_id in image_ids:
                        image = Image.objects.get(id=int(image_id))
                        if image.user_id != user_id:
                            return HttpResponse(status=status.HTTP_400_BAD_REQUEST)
                        if image.id == image_instance.id:
                            return HttpResponse(status=status.HTTP_403_FORBIDDEN)
                        current_identity_group = image.identity_group
                        for im in Image.objects.filter(identity_group=current_identity_group):
                            identical_images.append(im)
                    for im in identical_images:
                        im.identity_group = identity_group
                        im.save()
                    return HttpResponse(status=status.HTTP_204_NO_CONTENT)
                except Exception as exp:
                    logging.error(exp)
                    return HttpResponse(status=status.HTTP_400_BAD_REQUEST)
        return HttpResponse(status=status.HTTP_404_NOT_FOUND)
    return HttpResponse(status=status.HTTP_400_BAD_REQUEST)


import telebot
from telebot import types
from aiogram.types import ReplyKeyboardRemove, \
    ReplyKeyboardMarkup, KeyboardButton, \
    InlineKeyboardMarkup, InlineKeyboardButton

import os
import django

from backend import local2 as local
from worker.models import Worker, TGBotCode

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'backend.settings')
django.setup()

bot = telebot.TeleBot(os.environ.get('CHAT_BOT_TOKEN', local.CHAT_BOT_TOKEN))

def auth(message):
    msg = bot.send_message(message.chat.id, 'Введите код: ')
    bot.register_next_step_handler(msg, code_authorization)

def code_authorization(message):
    try:
        code = TGBotCode.objects.get(code=message.text)
    except TGBotCode.DoesNotExist:
        menu = get_menu(message)
        bot.send_message(
            message.chat.id,
            'Неверный код! Попробуйте авторизоваться еще раз!',
            reply_markup=menu
        )
    else:
        code.chat_id = message.chat.id
        code.save()
        menu = get_menu(message)
        bot.send_message(
            message.chat.id,
            f'Авторизация выполнена! Добро пожаловать, {code.user.get_full_name()}! ',
            reply_markup=menu
        )

def get_menu(message):
    menu = types.ReplyKeyboardMarkup(True, one_time_keyboard=True)
    try:
        TGBotCode.objects.get(chat_id=message.chat.id)
    except TGBotCode.DoesNotExist:
        menu.row('Авторизация')
        menu.row('Получить иформацию', 'Наставничество')
    else:
        menu.row('Выход')
        menu.row('Получить иформацию', 'Наставничество')
        menu.row('Моя информация', 'Мероприятия')
    finally:
        return menu

def make_keyboard(items, name):
    keyboard = types.InlineKeyboardMarkup()
    for item in items:
        keyboard.add(
            types.InlineKeyboardButton(
                item, callback_data=f'{name}:{item}'
            )
        )
    return keyboard


from Client0 import Client
from Seq1 import Seq
from colorama import Fore, init

init(autoreset=True)

PRACTICE = 2
EXERCISE = 5

print(f"-----| Practice {PRACTICE}, Exercise {EXERCISE} |------")

s1 = Seq()
filename = Seq.get_file2()
s1.read_fasta(filename)

frag1 = s1.strbases[:10]
frag2 = s1.strbases[11:20]
frag3 = s1.strbases[21:30]
frag4 = s1.strbases[31:40]
frag5 = s1.strbases[41:50]
frag6 = s1.strbases[51:60]
frag7 = s1.strbases[61:70]
frag8 = s1.strbases[71:80]
frag9 = s1.strbases[81:90]
frag10 = s1.strbases[91:100]

fragments_even = [frag2, frag4, frag6, frag8, frag10]
fragments_odd = [frag1, frag3, frag5, frag7, frag9]
i = 2
for f in fragments_even:
    # -- Parameters of the server to talk to
    IP = "127.0.0.1"
    PORT = 6123
    # -- Create a client object
    c = Client(IP, PORT)

    response = c.talk(Fore.YELLOW + f"Sending {i} gene to server")
    i = i + 2

p = 1
for f in fragments_odd:
    # -- Parameters of the server to talk to
    IP = "127.0.0.1"
    PORT = 8080
    # -- Create a client object
    c = Client(IP, PORT)

    response = c.talk(Fore.YELLOW + f"Sending fragment {p} gene to server")
    p = p + 2

s = 1
all_fragments = [frag1, frag2, frag3, frag4, frag5, frag6, frag7, frag8, frag9, frag10]
for t in all_fragments:
    print(f"Fragment {s}: {t}")
    s = s + 1

P2 done again/Session 9 - Practice 2/Exercise 6.py


import subprocess
import os
from unittest import mock
from unittest.mock import patch, Mock
import pytest
import logging
import sys

from src.build_java import build_java

def test_update_path_for_jdk():
    original_path = os.environ['PATH']
    build_java.update_path_for_jdk('java8')
    expected_path = '/usr/lib/jvm/java-1.8.0-openjdk-amd64/bin:' + original_path
    assert os.environ['PATH'] == expected_path

def test_build_with_maven(mocker):
    mocker.patch('subprocess.run', return_value=subprocess.CompletedProcess(args=[], returncode=0))
    mocker.patch('sys.argv', ['script.py', 'java11', 'clean install', ''])

    build_java.main()

    subprocess.run.assert_called_once_with(['mvn', 'clean', 'install', '-DskipFormModel=false', '-DskipTests', 'clean', 'install'])

def test_build_with_custom_command(mocker):
    mocker.patch('subprocess.run', return_value=subprocess.CompletedProcess(args=[], returncode=0))
    mocker.patch('sys.argv', ['script.py', 'java11', '', 'echo "Custom command"'])

    build_java.main()

    subprocess.run.assert_called_once_with('echo "Custom command"', shell=True)

def test_allow_sh_permissions():
    with patch('subprocess.run') as mock_run:
        build_java.allow_sh_permissions("script.sh")

        mock_run.assert_called_once_with(['chmod', '+x', 'script.sh'], check=True)

    with patch('subprocess.run', side_effect=subprocess.CalledProcessError(1, ['cmd'])):
        with pytest.raises(SystemExit) as e:
           build_java.allow_sh_permissions("script.sh")

        assert e.type == SystemExit
        assert e.value.code == 1


import matplotlib.pyplot as plt
import numpy as np
import matplotlib.animation

def getPerimeterPoints(targetPoint, S_inverse, angleStep, gateThreshold):
    """
        Input:
            targetPoint : np.array(shape = (dimX, 1))
            S_inverse : np.array(shape = (dimZ, dimZ))
            angleStep : float
    """

    x_draw = []
    y_draw = []

    angle = 0

    numberOfStep = int(np.pi * 2 / angleStep)

    for step in range(numberOfStep):

        angle = step * angleStep

        angleVector = np.array([np.cos(angle), np.sin(angle)]).reshape((2,1))

        R = np.sqrt(gateThreshold / np.dot(angleVector.T, np.dot(S_inverse, angleVector)))

        x_draw.append( targetPoint[0][0] + np.cos(angle) * R)
        y_draw.append( targetPoint[1][0] + np.sin(angle) * R)

    return (x_draw, y_draw)

def showPerimeter(targetPoint, S_inverse, angleStep, gateThreshold):
    """
        Input:
            targetPoint : np.array(shape = (dimX, 1))
            S_inverse : np.array(shape = (dimZ, dimZ))
            angleStep : float
    """

    x_draw, y_draw = getPerimeterPoints(targetPoint, S_inverse, angleStep, gateThreshold)

    plt.scatter(x_draw, y_draw, c = "m", alpha = 0.2, linewidths=1)
    plt.scatter([targetPoint[0][0]], [targetPoint[1][0]], c = "k", linewidths=3)

def showRadius(targetPoint, R, angleStep):
    """
        Input:
            targetPoint : np.array(shape = (dimX, 1))
            R : float
            angleStep : float
    """

    x_draw = []
    y_draw = []

    angle = 0

    numberOfStep = int(np.pi * 2 / angleStep)

    for step in range(numberOfStep):

        angle = step * angleStep

        x_draw.append( targetPoint[0][0] + np.cos(angle) * R)
        y_draw.append( targetPoint[1][0] + np.sin(angle) * R)

    plt.scatter(x_draw, y_draw, c = "k")
    plt.scatter([targetPoint[0][0]], [targetPoint[1][0]], c = "b", linewidths=3)

def visualizeTrackingResults(trackers, measurementPacks, groundTruthPacks, animate, gateThreshold):

    predictionPlotColor = ""
    measurementColor = ""

    if(animate):

        fig, ax = plt.subplots()
        ax.grid()

        drawingTrackers = []

        for tracker in trackers:
            drawingTrackers.append([0, [], None]) # historyIndex, historyUntilNow, lastPrediction( [z_prediction, S] )

        def animation(i):

            if not hasattr(animation, "time"):
                 animation.time = 0
            if not hasattr(animation, "mode"):
                 animation.mode = 0

            if not hasattr(animation, "perimeterX"):
                 animation.perimeterX = []
            if not hasattr(animation, "perimeterY"):
                 animation.perimeterY = []

            if not hasattr(animation, "centerXs"):
                 animation.centerXs = []
            if not hasattr(animation, "centerYs"):
                 animation.centerYs = []

            if not hasattr(animation, "measurementX"):
                 animation.measurementX = []
            if not hasattr(animation, "measurementY"):
                 animation.measurementY = []

            time = animation.time
            mode = animation.mode

            perimeterX = animation.perimeterX
            perimeterY = animation.perimeterY

            centerXs = animation.centerXs
            centerYs = animation.centerYs

            measurementX = animation.measurementX
            measurementY = animation.measurementY

            print(time)

            #print(time)
            #first clear screen
            ax.clear()

            if(mode == 0):

                centerXs = []
                centerYs = []

                measurementX = []
                measurementY = []

                perimeterX = []
                perimeterY = []

            #secondly draw the ground truths until now
            for groundTruthPack in groundTruthPacks:
                ax.plot(groundTruthPack[0], groundTruthPack[1])

            #thirdly draw all trackers until now

                #update the drawingTrackers
            for t, drawingTracker in enumerate(drawingTrackers):

                correspondingUpdateHistory = trackers[t].updatedStateHistory
                correspondingPredictHistory = trackers[t].predictedStateHistory

                if(drawingTracker[0] < len(correspondingUpdateHistory) and correspondingUpdateHistory[drawingTracker[0]][2] == time):

                    x_updated = correspondingUpdateHistory[drawingTracker[0]][0]

                    if(mode == 0):
                        drawingTracker[2] = (correspondingPredictHistory[drawingTracker[0]][0], correspondingPredictHistory[drawingTracker[0]][1])
                    elif(mode == 1):
                        drawingTracker[0] += 1
                        drawingTracker[1].append(x_updated)

                #now draw all updated states and new predictions
                #also stack the predictionPerimeters
            for drawingTracker in drawingTrackers:

                if(len(drawingTracker[1]) > 0):

                    states = np.array(drawingTracker[1])

                    ax.plot(states[:, 0], states[:, 1])

                    if(mode == 0):
                        # print("##############################\n \n")
                        # print("1")
                        predictionZ, predictionS = drawingTracker[2]
                        # print("2")

                        x_perimeter_draw, y_perimeter_draw = getPerimeterPoints(predictionZ, np.linalg.inv(predictionS), np.pi/180, gateThreshold)
                        # print("3")

                        perimeterX += x_perimeter_draw
                        perimeterY += y_perimeter_draw
                        # print("4")

                        centerX = []
                        centerY = []
                        # print("5")

                        centerX.append(states[-1, 0][0])
                        centerY.append(states[-1, 1][0])
                        # print("6")

                        centerX.append(predictionZ[0][0])
                        centerY.append(predictionZ[1][0])
                        # print("7")

                        # print(centerX)
                        # print(centerY)

                        centerXs.append(centerX)
                        centerYs.append(centerY)

                        # print("8\n\n")

            # print("before update measurements")
            #update measurements if mode is 0
            if(mode == 0): #update the measurements

                measurementX = []
                measurementY = []

                measurementPack = measurementPacks[time]

                for measurement in measurementPack:
                    measurementX.append(measurement[0])
                    measurementY.append(measurement[1])

            # print("before draw measurements")
            #draw measurements
            ax.scatter(measurementX, measurementY, c = "g")

            #draw perimeters and centers
            ax.scatter(perimeterX, perimeterY, c = "m")

            # print("before draw centerX centerY")
            for centerX, centerY in zip(centerXs, centerYs):
                ax.scatter(centerX[-1], centerY[-1], c = "k")
                ax.plot(centerX, centerY, c = '#07516e', linewidth = 5.0)

            # print("before auto scaling")

            #finally auto scaling the screen
            if(time == -1):
                ax.relim()
                ax.autoscale_view(True,True,True)

            if(mode == 1):
                time += 1

            mode = (mode + 1) % 2

            animation.mode = mode
            animation.time = time

            animation.perimeterX = perimeterX
            animation.perimeterY = perimeterY

            animation.centerXs = centerXs
            animation.centerYs = centerYs

            animation.measurementX = measurementX
            animation.measurementY = measurementY

            #end animation function

        ani = matplotlib.animation.FuncAnimation(fig, animation,
                frames=1000, interval=100, repeat=False)

        plt.show()

        return ani
    else:
        print("todo")

        for tracker in trackers:

            predictions = tracker.updatedStateHistory

            xs = []
            Ps = []

            for prediction in predictions:
                xs.append(prediction[0])
                Ps.append(prediction[1])

            xs = np.array(xs)
            Ps = np.array(Ps)

            plt.plot(xs[:,0], xs[:,1])
            showPerimeter(xs[-1][0:2], np.linalg.inv(Ps[-1][0:2,0:2]), np.pi/100, gateThreshold)


import logging
from conftest import Log_path

class LogData:
    def getLogger(self):
        logger = logging.getLogger(__name__)
        logsone = logging.FileHandler(Log_path+"/logfile.log")
        formatter = logging.Formatter("%(asctime)s :%(levelname)s :%(name)s :%(message)s")
        logsone.setFormatter(formatter)

        logger.addHandler(logsone)

        logger.setLevel(logging.DEBUG)
        return logger


import pandas as pd
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
import os
import time
from datetime import date, datetime
from datetime import timedelta
def wait_for_downloads(path):
    print("Waiting for downloads", end="")
    while any([filename.endswith(".crdownload") for filename in
               os.listdir(path)]):
        time.sleep(1)
        print(".", end="")
    print("done!")

downloadPath = os.path.abspath('./../database/sp500')
os.makedirs(downloadPath, mode=0o777, exist_ok=True)  # create dir if it not exist

options = webdriver.ChromeOptions()

options.add_argument('--headless')
options.add_argument('--disable-gpu')
options.add_argument("--allow-running-insecure-content")
options.add_argument("--window-size=1920,1080")
options.add_argument('--no-sandbox')
options.add_argument(f'user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36')
options.add_argument("--remote-debugging-address=0.0.0.0")
options.add_argument("--remote-debugging-port=9222")
options.add_argument("--disable-features=VizDisplayCompositor")
options.add_argument("--disable-dev-shm-usage")
options.add_argument("--disable-infobars")
options.add_argument("--disable-browser-side-navigation")
options.add_experimental_option('prefs', {
"download.default_directory": downloadPath, #Change default directory for downloads
"download.prompt_for_download": False, #To auto download the file
"download.directory_upgrade": True,
"plugins.always_open_pdf_externally": True #It will not show PDF directly in chrome
})

print("Init chrome driver ...")
driver = webdriver.Chrome(service = Service(executable_path= '/usr/local/bin/chromedriver'),options=options)
driver.implicitly_wait(10)
print("chrome driver ok!")

url = "https://finance.yahoo.com/quote/%5EGSPC/history?p=%5EGSPC"

driver.get(url)

tabela = driver.find_element(By.CSS_SELECTOR, "#Col1-1-HistoricalDataTable-Proxy > section > div.Pb\(10px\).Ovx\(a\).W\(100\%\) > table")
df = pd.read_html(tabela.get_attribute("outerHTML"))[0]
df = df[:-1]
df['Date'] = [datetime.strptime(str(d), "%b %d, %Y") for d in df['Date']]

if datetime.now().weekday() == 0:#segunda
    data = datetime.strftime((datetime.now() - timedelta(days=3)).date(), "%Y-%m-%d")
    dataAnterior = datetime.strftime((datetime.now() - timedelta(days=4)).date(), "%Y-%m-%d")
elif datetime.now().weekday() == 1:#terça
    data = datetime.strftime((datetime.now() - timedelta(days=1)).date(), "%Y-%m-%d")
    dataAnterior = datetime.strftime((datetime.now() - timedelta(days=4)).date(), "%Y-%m-%d")
else:
    data = datetime.strftime((datetime.now() - timedelta(days=1)).date(), "%Y-%m-%d")
    dataAnterior = datetime.strftime((datetime.now() - timedelta(days=2)).date(), "%Y-%m-%d")

valorAntigo = df.loc[df['Date'] == dataAnterior, "Adj Close**"].values[0]
valor = df.loc[df['Date'] == data, "Adj Close**"].values[0]
retorno = [float(valor)/float(valorAntigo) - 1]
data = [data]
d = {"Data":data, "Valor":retorno}
finalDf = pd.DataFrame(data=d)

finalDf.to_csv(os.path.join(downloadPath, "sp500.csv"),  sep = ",", index=False)


"""

create table product (name varchar(20) primary key, price float, qty int)
drop table product # to delete the whole table

insert into product values ('Prod1', 34.56, 23)
select * from product
select * from product where qty > 20
delete from product where qty=0
update product set price=price*1.25 where name='Prod3'

commit # validate a transaction
rollback # invalidate the transaction (the insert,delete,update done previously)

sql alchemy

"""

import sqlite3

try:
    # Step 1: connection
    conn=sqlite3.connect(r"epfl.db")
    # Step 2: create a cursor object
    cursor=conn.cursor()
    # Step 3: using the cursor, execute SQL statements:
    cursor.execute("insert into product values ('prod600', 22.3, 34)")
    print(f"{cursor.rowcount} rows were returned")
    cursor.execute("commit")

    cursor.execute("select * from product")

    while True:
        row = cursor.fetchone()
        if row == None:
            break
#         print("{}, {}, {}".format(row[0], row[1], row[2]))
        print(f"{row[0]}, {row[1]}, {row[2]}")
#     res=cursor.fetchall()
#     print(res)
    print("{} rows were returned".format(cursor.rowcount))

    cursor.close()
    conn.close()
except Exception as ex:
    print(ex)


import os
import json
import io
from wp import wp_database, wp_post_query
from mailing import mailer

WP_DB_CONFIG_FILE_PATH_VAR_KEY = "WP_DB_CONFIG_FILE_PATH"
DEFAULT_WP_DB_CONFIG_FILE_PATH = "resource/wp_db_config.json"

MAIL_CONFIG_FILE_PATH_VAR_KEY = "MAIL_CONFIG_FILE_PATH"
DEFAULT_MAIL_CONFIG_FILE_PATH = "resource/mail_config.json"

JINJA2_MAIL_TEMPLATE_FILE_PATH_VAR_KEY ='JINJA2_MAIL_TEMPLATE_FILE_PATH'
DEFAULT_JINJA2_MAIL_TEMPLATE_FILE_PATH = 'resource/mail_template.jinja2'

# WordPress
def get_wp_db_config():
    wpDbConfigFilePath = get_wp_db_config_file_path()
    wpDbConfig = get_json_data(wpDbConfigFilePath)
    wp_database.validate_wp_db_config(wpDbConfig)
    return wpDbConfig

def get_wp_db_config_file_path():
    return get_env_config(WP_DB_CONFIG_FILE_PATH_VAR_KEY, DEFAULT_WP_DB_CONFIG_FILE_PATH)

# Mail
def get_mail_config():
    mailConfigFilePath = get_mail_config_file_path()
    mailConfig = get_json_data(mailConfigFilePath)
    mailer.validate_mail_config(mailConfig)
    return mailConfig

def get_mail_config_file_path():
    return get_env_config(MAIL_CONFIG_FILE_PATH_VAR_KEY, DEFAULT_MAIL_CONFIG_FILE_PATH)

# Jinja 2 Template
def get_jinja2_mail_template_string():
    filePath = get_jinja2_mail_template_file_path()
    with io.open(filePath, "r") as jinja2_file:
        return jinja2_file.read()

def get_jinja2_mail_template_file_path():
    return get_env_config(JINJA2_MAIL_TEMPLATE_FILE_PATH_VAR_KEY, DEFAULT_JINJA2_MAIL_TEMPLATE_FILE_PATH)

# Config
def get_env_config(key, default):
    return default if not os.environ.get(key) else os.environ.get(key)

def get_json_data(filename):
    with io.open(filename, "r", encoding="utf-8") as json_file:
        data = json.load(json_file)
        return data

# Main
def main():
    wpDbConfig  = get_wp_db_config()
    mailConfig = get_mail_config()
    jinja2MailTemplateString = get_jinja2_mail_template_string()

    db = wp_database.get_wp_db(wpDbConfig)

    queryResult = wp_post_query.get_pending_posts(
        db=db,
        targetCategories=wpDbConfig['targetCategories'],
        targetPostStatuses=wpDbConfig['targetPostStatuses']
    )

    print(queryResult)

    mailer.send(mailConfig, queryResult, jinja2MailTemplateString)

if __name__ == '__main__':
    main()


from typing import List

# lapsolver is quite hard to install on Windows. Fortunately, scipy offers a similar, albeit slower, function.
try:
    from lapsolver import solve_dense as solver
except ImportError:
    from scipy.optimize import linear_sum_assignment as solver

from authentication.models import User
from repartitions.models import (
    Proposition,
    Wish,
    Category,
    Group,
    Campaign,
    UserCampaign,
)

import numpy as np

"""
The repartition algorithm is based on Munkres algorithm, or "Hungarian" algorithm, which, given a cost matrix,
returns a pairing between lines and columns that minimises the cost.

The implementation we use here is lapsolver's
https://github.com/cheind/py-lapsolver

However it does more than simply calling Munkres algorithm because we have two additional constraints :
- we have  "categories" of users ie subsets that have to be mixed inside of each group. For instance if we have 10 users
"A" and 10 users "B" to split into two groups we want to have twice "5 A  + 5 B" instead of one group of all As and one
group of all Bs
- we want groups to be "even" ie we don't want to be more than 1 person of difference between two groups (ex : one group
of 15 people and another of 5 people) nor inside of a group between categories (ex: 2A and 8B in one group and 8A and 2B
 in the other)

To abide by theses constraints, the algorithm is split in 2 stages:
(1) make an even repartition for a category
    determine the number of place necessary for each group
        - if there are n people and m groups, we'll have k places in each group were (k-1) * m < n <= k * m

    create the cost matrix
        - one line by user
        - one column per place in group, each column corresponding to one group is identical
        - example : 3 users, and 2 groups => k = 4
            (1, 1) (1, 1) (1, 2) (1, 2)
            (2, 1) (2, 1) (2, 2) (2, 2)
            (3, 1) (3, 1) (3, 2) (3, 2)

    run hungarian algorithm

    if there are too many people in one or several group, add a "penalty" to the group which has the most people. This
    is as if one place was removed in this group so on the next iteration it'll have one less person

    rerun the algorithm as long as the repartition isn't even

(2)  make the repartition even among categories
    repeat step (1) for each category but adding a penalty on each run corresponding to how many people each group has
    more in cumulated (either 0 or 1)

"""

def generate_line(
    uc: UserCampaign, propositions: List[Proposition], places: List[int]
) -> List[float]:
    """given a user's choices, a list of proposition and the number of places for each proposition, returns the
    corresponding line of the cost matrix"""

    if any(map(lambda x: x < 0, places)):
        raise ValueError("places {} is not valid", places)

    wishes = Wish.objects.filter(user_campaign=uc).all()
    map_wishes = {w.proposition.id: w for w in wishes}

    total_number_of_places = 0
    for n in places:
        total_number_of_places += n

    res = [0] * total_number_of_places
    i = 0
    for j, proposition in enumerate(propositions):
        for k in range(places[j]):
            if proposition.id in map_wishes:
                res[i] = map_wishes.get(proposition.id).rank
            else:
                res[i] = 0
            i += 1

    return res

def get_project_index(index, places):
    """Given an index and the number of places in each proposition, returns the index of the proposition in the
    array given to `generate_line`"""
    s = 0
    for i in range(len(places)):
        if s <= index < s + places[i]:
            return i
        s += places[i]

    if index == s:
        return len(places) - 1

    raise ValueError("index is too big considering the number of places")

# noinspection PyTypeChecker
def make_repartition_for_category(
    category: Category, propositions: List[Proposition], already_used: List[int]
) -> List[List[UserCampaign]]:
    raw_users_campaign = category.users_campaign.all()

    if len(raw_users_campaign) == 0:
        raise ValueError("No users in given group")
    user_campaigns, fixed_user_campaigns = [], []
    for uc in raw_users_campaign:
        if not uc.fixed_to:
            user_campaigns.append(uc)
        else:
            fixed_user_campaigns.append(uc)

    repartition_cardinal_diff = 2
    penalty = [0] * len(already_used)
    while repartition_cardinal_diff > 1:
        repartition = [[] for _ in propositions]
        proposition_id_to_index = {
            proposition.id: i for i, proposition in enumerate(propositions)
        }

        if len(fixed_user_campaigns) > 0:
            for uc in fixed_user_campaigns:
                repartition[proposition_id_to_index[uc.fixed_to.id]].append(uc)

        if len(user_campaigns) > 0:
            places = []
            s = 0  # Nombre de places allouées pour le groupe
            for i, proposition in enumerate(propositions):
                n = -already_used[i]
                n -= len(repartition)
                n -= penalty[i]
                s += n
                places.append(n)

            while s < len(user_campaigns):
                for i in range(len(places)):
                    s += 1
                    places[i] += 1

            cost_matrix = []
            for uc in user_campaigns:
                cost_matrix.append(generate_line(uc, propositions, places))

            # make the matrix square
            while len(cost_matrix) < len(cost_matrix[0]):
                cost_matrix.append([0] * len(cost_matrix[0]))

            links = solver(cost_matrix)

            for user_index, place_index in zip(links[0], links[1]):
                if user_index >= len(user_campaigns):
                    break

                project_index = get_project_index(place_index, places)
                uc = user_campaigns[user_index]
                repartition[project_index].append(uc)

            repartition_size = list(map(len, repartition))
            repartition_cardinal_diff = max(repartition_size) - min(repartition_size)

            penalty[np.argmax(repartition_size)] += 1
        else:
            repartition_cardinal_diff = (
                0  # there is nothing to do, avoid being stuck in an infinite loop
            )

    return repartition

def make_reparitition_proxy(
    category: Category, propositions: List[Proposition], already_used: List[int]
) -> List[List[User]]:
    """makes repartition for one category and make sure it's even"""
    group_cardinal_diff = 2
    groups = []

    already_used = np.array(already_used)
    penalty = np.array([0] * len(already_used))
    while group_cardinal_diff > 1:
        used_places = already_used + penalty
        groups = make_repartition_for_category(category, propositions, used_places)

        rep = [0] * len(used_places)
        for i in range(len(propositions)):
            rep[i] = len(groups[i])
        group_cardinal = np.array(rep) + already_used
        penalty[np.argmax(group_cardinal)] += 1
        group_cardinal_diff = max(group_cardinal) - min(group_cardinal)

    return groups

def make_reparition(campaign: Campaign) -> List[Group]:
    propositions = list(campaign.propositions.all())
    categories = list(campaign.categories.all())

    already_used = [0] * len(propositions)
    store = {}

    for i in range(len(categories)):
        category = categories[i]

        if category.users_campaign.count() == 0:
            continue

        # Check the repartition is feasible
        user_campaigns = category.users_campaign.all()
        fixed_to = {}
        for uc in user_campaigns:
            if uc.fixed_to is not None:
                fixed_to[uc.fixed_to.id] = fixed_to.get(uc.fixed_to.id, 0) + 1

        # if it's not possible for the repartition to compensate the inequalities created by fixity
        if len(fixed_to) > 0 and (max(fixed_to.values()) - 1) * len(propositions) > len(
            user_campaigns
        ):
            raise RuntimeError(
                "Impossible to find an even repartition, {} out of {} users fixed to {}".format(
                    max(fixed_to.values()),
                    category.users_campaign.count(),
                    np.argmax(fixed_to),
                )
            )

        # dispatch users
        groups = make_reparitition_proxy(category, propositions, already_used)

        for j, proposition in enumerate(propositions):
            store[proposition.id] = store.get(proposition.id, []) + groups[j]
            already_used[j] += len(groups[j])

    groups = []
    for proposition_id, users in store.items():

        proposition = Proposition.objects.get(pk=proposition_id)
        group = Group(proposition=proposition, campaign=campaign)
        group.save()
        group.users.set(users)
        groups.append(group)

    return groups