from pyspark.sql import SparkSession
spark = SparkSession.builder.getOrCreate()
spark.conf.set('spark.sql.parquet.compression.codec', 'snappy')
spark.conf.set('hive.exec.dynamic.partition.mode', 'nonstrict')
spark.conf.set('spark.streaming.stopGracefullyOnShutdown', 'true')
spark.conf.set('hive.exec.max.dynamic.partitions', '3000')
spark.conf.set('hive.support.concurrency', 'true')

from pyspark.sql import functions as f
from pyspark.sql import types as t

# variables globales

class Modelacion_02_feat():

    def __init__(self):
        self.str1='First Class'

    def export_table(self,TRAIN_POB_CAP,VAR_MES):

        # Lectura en el server
        datos_contacto = spark.read.table("cd_baz_bdclientes.cd_cte_datos_contacto_master") \
            .select(
                f.col('id_master'),
                f.col('lentidad'),
                f.col('genero'),
                f.col('fecha_nacimiento'),
                f.col('cposta').alias('cod_postal')) \
            .withColumn('entidad', f.when(f.trim(f.col('lentidad')).isin('VERACRUZ', 'VERACRUZ DE IGNACIO DE LA LLAVE'), 'VERACRUZ') \
            .otherwise(f.trim(f.col('lentidad')))) \
            .drop(f.col('lentidad'))

        recorrido = spark.read.table("cd_baz_bdclientes.cd_con_cte_recorrido") \
            .select(
                f.col('id_master'),
                f.col('num_periodo_mes').alias('per_ref'),
                f.col('cod_perfil_trx'),
                f.col('saldo'),
                f.col('potencial'),
                f.col('recorrido')) \
            .filter(f.col('per_ref') == str(VAR_MES)) \
            .orderBy(f.col('id_master'))

        # Secuencia de extracion de tablas
        TT_train_feat_ren_ind =  self.feat_cap(recorrido,datos_contacto,VAR_MES,TRAIN_POB_CAP)
        respond = TT_train_feat_ren_ind
        return  respond

    # Paso 1: Extraccion de informacion para el modelo de potenciales
    def feat_cap(self,recorrido,datos_contacto,VAR_MES,TRAIN_POB_CAP):

        _sdm = \
            datos_contacto.alias('A').withColumn('genero', f.when(f.trim(f.col('genero')).isin('N', 'E'), 'X') \
                .otherwise(f.col('genero'))) \
                .withColumn('var_mes', f.to_date(f.lit(str(VAR_MES)+'01'), 'yyyyMMdd')) \
                .withColumn('edad', f.round(f.months_between(f.col('var_mes'), f.col('fecha_nacimiento')) / 12, 0).cast(t.IntegerType())) \
                .select(
                    f.col('id_master'),
                    f.col('edad'),
                    f.col('var_mes'),
                    f.col('genero'),
                    f.col('cod_postal'),
                    f.col('entidad')) \
                .orderBy('id_master')

        TT_train_feat_ren_ind = \
            TRAIN_POB_CAP.alias('A').join(_sdm.alias('B'), f.col('A.id_master') == f.col('B.id_master'), 'left') \
                .join(recorrido.alias('D'), f.col('A.id_master') == f.col('D.id_master'), 'left') \
                .select(
                    f.col('A.id_master'),
                    f.col('A.per_ref'),
                    f.col('A.mto_ing_mes'),
                    f.coalesce(f.col('B.genero'), f.lit('VACIO')).alias('genero'),
                    f.coalesce(f.col('B.edad'), f.lit(0)).alias('edad'), # mayor a 18
                    f.coalesce(f.col('B.entidad'), f.lit('VACIO')).alias('entidad'),
                    f.coalesce(f.col('B.cod_postal'), f.lit(0)).alias('cod_postal'),
                    f.coalesce(f.col('D.saldo'), f.lit(0)).alias('saldo'),
                    f.coalesce(f.col('D.potencial'), f.lit(0)).alias('potencial'),
                    f.coalesce(f.col('D.recorrido'), f.lit(0)).alias('recorrido')) \
                .orderBy('id_master')

        del datos_contacto
        del recorrido
        del _sdm

        return TT_train_feat_ren_ind


import datetime
import urllib
import urllib.parse
from mpcomp import http_core

try:
    import simplejson
    from simplejson.decoder import JSONDecodeError
except ImportError:
    JSONDecodeError = None
    try:
        # Try to import from django, should work on App Engine
        from django.utils import simplejson
    except ImportError:
        # Should work for Python2.6 and higher.
        import json as simplejson

__author__ = "j.s@google.com (Jeff Scudder)"

PROGRAMMATIC_AUTH_LABEL = "GoogleLogin auth="
AUTHSUB_AUTH_LABEL = "AuthSub token="
OAUTH2_AUTH_LABEL = "Bearer "

# This dict provides the AuthSub and OAuth scopes for all services by service
# name. The service name (key) is used in ClientLogin requests.
AUTH_SCOPES = {
    "cl": (  # Google Calendar API
        "https://www.google.com/calendar/feeds/",
        "http://www.google.com/calendar/feeds/",
    ),
    "gbase": (  # Google Base API
        "http://base.google.com/base/feeds/",
        "http://www.google.com/base/feeds/",
    ),
    "blogger": ("http://www.blogger.com/feeds/",),  # Blogger API
    "codesearch": (  # Google Code Search API
        "http://www.google.com/codesearch/feeds/",
    ),
    "cp": (  # Contacts API
        "https://www.google.com/m8/feeds/",
        "http://www.google.com/m8/feeds/",
    ),
    "finance": ("http://finance.google.com/finance/feeds/",),  # Google Finance API
    "health": ("https://www.google.com/health/feeds/",),  # Google Health API
    "writely": (  # Documents List API
        "https://docs.google.com/feeds/",
        "https://spreadsheets.google.com/feeds/",
        "https://docs.googleusercontent.com/",
    ),
    "lh2": ("http://picasaweb.google.com/data/",),  # Picasa Web Albums API
    "apps": (  # Google Apps Domain Info & Management APIs
        "https://apps-apis.google.com/a/feeds/user/",
        "https://apps-apis.google.com/a/feeds/policies/",
        "https://apps-apis.google.com/a/feeds/alias/",
        "https://apps-apis.google.com/a/feeds/groups/",
        "https://apps-apis.google.com/a/feeds/compliance/audit/",
        "https://apps-apis.google.com/a/feeds/migration/",
        "https://apps-apis.google.com/a/feeds/emailsettings/2.0/",
    ),
    "weaver": ("https://www.google.com/h9/feeds/",),  # Health H9 Sandbox
    "wise": ("https://spreadsheets.google.com/feeds/",),  # Spreadsheets Data API
    "sitemaps": (  # Google Webmaster Tools API
        "https://www.google.com/webmasters/tools/feeds/",
    ),
    "youtube": (  # YouTube API
        "http://gdata.youtube.com/feeds/api/",
        "http://uploads.gdata.youtube.com/feeds/api",
        "http://gdata.youtube.com/action/GetUploadToken",
    ),
    "books": ("http://www.google.com/books/feeds/",),  # Google Books API
    "analytics": ("https://www.google.com/analytics/feeds/",),  # Google Analytics API
    "jotspot": (  # Google Sites API
        "http://sites.google.com/feeds/",
        "https://sites.google.com/feeds/",
    ),
    # "local": ("http://maps.google.com/maps/feeds/",),  # Google Maps Data API
    "code": ("http://code.google.com/feeds/issues",),  # Project Hosting Data API
}

class Error(Exception):
    pass

class UnsupportedTokenType(Error):

    """Raised when token to or from blob is unable to convert the token."""

    pass

class OAuth2AccessTokenError(Error):

    """Raised when an OAuth2 error occurs."""

    def __init__(self, error_message):
        self.error_message = error_message

class OAuth2RevokeError(Error):

    """Raised when an OAuth2 token revocation was unsuccessful."""

    def __init__(self, http_response, response_body=None):
        """Sets the HTTP information in the error.

        Args:
          http_response: The response from the server, contains error information.
          response_body: string (optional) specified if the response has already
                         been read from the http_response object.
        """
        body = response_body or http_response.read()

        self.status = http_response.status
        self.reason = http_response.reason
        self.body = body
        self.headers = http_core.get_headers(http_response)

        self.error_msg = "Invalid response %s." % self.status
        try:
            json_from_body = simplejson.loads(body)
            if isinstance(json_from_body, dict):
                self.error_msg = json_from_body.get("error", self.error_msg)
        except (ValueError, JSONDecodeError):
            pass

    def __str__(self):
        return "OAuth2RevokeError(status=%i, error=%s)" % (self.status, self.error_msg)

REQUEST_TOKEN = 1
AUTHORIZED_REQUEST_TOKEN = 2
ACCESS_TOKEN = 3

class OAuth2Token(object):

    """Token object for OAuth 2.0 as described on
    <http://code.google.com/apis/accounts/docs/OAuth2.html>.

    Token can be applied to a gdata.client.GDClient object using the authorize()
    method, which then signs each request from that object with the OAuth 2.0
    access token.
    This class supports 3 flows of OAuth 2.0:
      Client-side web flow: call generate_authorize_url with `response_type='token''
        and the registered `redirect_uri'.
      Server-side web flow: call generate_authorize_url with the registered
        `redirect_url'.
      Native applications flow: call generate_authorize_url as it is. You will have
        to ask the user to go to the generated url and pass in the authorization
        code to your application.
    """

    def __init__(
        self,
        client_id,
        client_secret,
        scope,
        user_agent,
        auth_uri="https://accounts.google.com/o/oauth2/auth",
        token_uri="https://accounts.google.com/o/oauth2/token",
        access_token=None,
        refresh_token=None,
        revoke_uri="https://accounts.google.com/o/oauth2/revoke",
    ):
        """Create an instance of OAuth2Token

        Args:
          client_id: string, client identifier.
          client_secret: string client secret.
          scope: string, scope of the credentials being requested.
          user_agent: string, HTTP User-Agent to provide for this application.
          auth_uri: string, URI for authorization endpoint. For convenience
            defaults to Google's endpoints but any OAuth 2.0 provider can be used.
          token_uri: string, URI for token endpoint. For convenience
            defaults to Google's endpoints but any OAuth 2.0 provider can be used.
          revoke_uri: string, URI for revoke endpoint. For convenience
            defaults to Google's endpoints but any OAuth 2.0 provider can be used.
          access_token: string, access token.
          refresh_token: string, refresh token.
        """
        self.client_id = client_id
        self.client_secret = client_secret
        self.scope = scope
        self.user_agent = user_agent
        self.auth_uri = auth_uri
        self.token_uri = token_uri
        self.revoke_uri = revoke_uri
        self.access_token = access_token
        self.refresh_token = refresh_token

        # True if the credentials have been revoked or expired and can't be
        # refreshed.
        self._invalid = False

    @property
    def invalid(self):
        """True if the credentials are invalid, such as being revoked."""
        return getattr(self, "_invalid", False)

    def _refresh(self, request):
        """Refresh the access_token using the refresh_token.

        Args:
          request: The atom.http_core.HttpRequest which contains all of the
              information needed to send a request to the remote server.
        """
        body = urllib.parse.urlencode(
            {
                "grant_type": "refresh_token",
                "client_id": self.client_id,
                "client_secret": self.client_secret,
                "refresh_token": self.refresh_token,
            }
        )
        headers = {
            "user-agent": self.user_agent,
        }

        http_request = http_core.HttpRequest(
            uri=self.token_uri, method="POST", headers=headers
        )
        http_request.add_body_part(body, mime_type="application/x-www-form-urlencoded")
        response = request(http_request)
        body = response.read()
        if response.status == 200:
            self._extract_tokens(body)
        else:
            self._invalid = True
        return response

    def _extract_tokens(self, body):
        d = simplejson.loads(body)
        self.access_token = d["access_token"]
        self.refresh_token = d.get("refresh_token", self.refresh_token)
        if "expires_in" in d:
            self.token_expiry = (
                datetime.timedelta(seconds=int(d["expires_in"]))
                + datetime.datetime.now()
            )
        else:
            self.token_expiry = None

    def authorize(self, client):
        """Authorize a gdata.client.GDClient instance with these credentials.

        Args:
           client: An instance of gdata.client.GDClient
               or something that acts like it.

        Returns:
           A modified instance of client that was passed in.

        Example:
          >>> c = gdata.client.GDClient(source='user-agent')
          >>> c = token.authorize(c)
        """
        client.auth_token = self
        request_orig = client.http_client.request

        def new_request(http_request):
            response = request_orig(http_request)
            if response.status == 401:
                refresh_response = self._refresh(request_orig)
                if self._invalid:
                    return refresh_response
                self.modify_request(http_request)
                return request_orig(http_request)
            return response

        client.http_client.request = new_request
        return client

    def modify_request(self, http_request):
        """Sets the Authorization header in the HTTP request using the token.

        Returns:
          The same HTTP request object which was passed in.
        """
        http_request.headers["Authorization"] = "%s%s" % (
            OAUTH2_AUTH_LABEL,
            self.access_token,
        )
        return http_request

    ModifyRequest = modify_request

def _make_credentials_property(name):
    """Helper method which generates properties.

    Used to access and set values on credentials property as if they were native
    attributes on the current object.

    Args:
      name: A string corresponding to the attribute being accessed on the
          credentials attribute of the object which will own the property.

    Returns:
      An instance of `property` which is a proxy for the `name` attribute on the
          credentials attribute of the object.
    """

    def get_credentials_value(self):
        return getattr(self.credentials, name)

    def set_credentials_value(self, value):
        setattr(self.credentials, name, value)

    return property(get_credentials_value, set_credentials_value)


import io
from typing import List, Set, Tuple

from clvm import KEYWORD_FROM_ATOM, KEYWORD_TO_ATOM, SExp
from clvm import run_program as default_run_program
from clvm.casts import int_from_bytes
from clvm.EvalError import EvalError
from clvm.operators import OP_REWRITE, OPERATOR_LOOKUP
from clvm.serialize import sexp_from_stream, sexp_to_stream
from clvm_rs import STRICT_MODE, deserialize_and_run_program2, serialized_length
from clvm_tools.curry import curry, uncurry

from chia.types.blockchain_format.sized_bytes import bytes32
from chia.util.hash import std_hash

from .tree_hash import sha256_treehash

def run_program(
    program,
    args,
    max_cost,
    operator_lookup=OPERATOR_LOOKUP,
    pre_eval_f=None,
):
    return default_run_program(
        program,
        args,
        operator_lookup,
        max_cost,
        pre_eval_f=pre_eval_f,
    )

INFINITE_COST = 0x7FFFFFFFFFFFFFFF

class Program(SExp):
    """
    A thin wrapper around s-expression data intended to be invoked with "eval".
    """

    @classmethod
    def parse(cls, f) -> "Program":
        return sexp_from_stream(f, cls.to)

    def stream(self, f):
        sexp_to_stream(self, f)

    @classmethod
    def from_bytes(cls, blob: bytes) -> "Program":
        f = io.BytesIO(blob)
        result = cls.parse(f)  # type: ignore # noqa
        assert f.read() == b""
        return result

    def to_serialized_program(self) -> "SerializedProgram":
        return SerializedProgram.from_bytes(bytes(self))

    def __bytes__(self) -> bytes:
        f = io.BytesIO()
        self.stream(f)  # type: ignore # noqa
        return f.getvalue()

    def __str__(self) -> str:
        return bytes(self).hex()

    def get_tree_hash(self, *args: List[bytes32]) -> bytes32:
        """
        Any values in `args` that appear in the tree
        are presumed to have been hashed already.
        """
        return sha256_treehash(self, set(args))

    def run_with_cost(self, max_cost: int, args) -> Tuple[int, "Program"]:
        prog_args = Program.to(args)
        cost, r = run_program(self, prog_args, max_cost)
        return cost, Program.to(r)

    def run(self, args) -> "Program":
        cost, r = self.run_with_cost(INFINITE_COST, args)
        return r

    def curry(self, *args) -> "Program":
        cost, r = curry(self, list(args))
        return Program.to(r)

    def uncurry(self) -> Tuple["Program", "Program"]:
        r = uncurry(self)
        if r is None:
            return self, self.to(0)
        return r

    def as_int(self) -> int:
        return int_from_bytes(self.as_atom())

    def as_atom_list(self) -> List[bytes]:
        """
        Pretend `self` is a list of atoms. Return the corresponding
        python list of atoms.

        At each step, we always assume a node to be an atom or a pair.
        If the assumption is wrong, we exit early. This way we never fail
        and always return SOMETHING.
        """
        items = []
        obj = self
        while True:
            pair = obj.pair
            if pair is None:
                break
            atom = pair[0].atom
            if atom is None:
                break
            items.append(atom)
            obj = pair[1]
        return items

    def __deepcopy__(self, memo):
        return type(self).from_bytes(bytes(self))

    EvalError = EvalError

def _tree_hash(node: SExp, precalculated: Set[bytes32]) -> bytes32:
    """
    Hash values in `precalculated` are presumed to have been hashed already.
    """
    if node.listp():
        left = _tree_hash(node.first(), precalculated)
        right = _tree_hash(node.rest(), precalculated)
        s = b"\2" + left + right
    else:
        atom = node.as_atom()
        if atom in precalculated:
            return bytes32(atom)
        s = b"\1" + atom
    return bytes32(std_hash(s))

def _serialize(node) -> bytes:
    if type(node) == SerializedProgram:
        return bytes(node)
    else:
        return SExp.to(node).as_bin()

class SerializedProgram:
    """
    An opaque representation of a clvm program. It has a more limited interface than a full SExp
    """

    _buf: bytes = b""

    @classmethod
    def parse(cls, f) -> "SerializedProgram":
        length = serialized_length(f.getvalue()[f.tell() :])
        return SerializedProgram.from_bytes(f.read(length))

    def stream(self, f):
        f.write(self._buf)

    @classmethod
    def from_bytes(cls, blob: bytes) -> "SerializedProgram":
        ret = SerializedProgram()
        ret._buf = bytes(blob)
        return ret

    @classmethod
    def from_program(cls, p: Program) -> "SerializedProgram":
        ret = SerializedProgram()
        ret._buf = bytes(p)
        return ret

    def to_program(self) -> Program:
        return Program.from_bytes(self._buf)

    def uncurry(self) -> Tuple["Program", "Program"]:
        return self.to_program().uncurry()

    def __bytes__(self) -> bytes:
        return self._buf

    def __str__(self) -> str:
        return bytes(self).hex()

    def __repr__(self):
        return "%s(%s)" % (self.__class__.__name__, str(self))

    def __eq__(self, other) -> bool:
        if not isinstance(other, SerializedProgram):
            return False
        return self._buf == other._buf

    def __ne__(self, other) -> bool:
        if not isinstance(other, SerializedProgram):
            return True
        return self._buf != other._buf

    def get_tree_hash(self, *args: List[bytes32]) -> bytes32:
        """
        Any values in `args` that appear in the tree
        are presumed to have been hashed already.
        """
        tmp = sexp_from_stream(io.BytesIO(self._buf), SExp.to)
        return _tree_hash(tmp, set(args))

    def run_safe_with_cost(self, max_cost: int, *args) -> Tuple[int, Program]:
        return self._run(max_cost, STRICT_MODE, *args)

    def run_with_cost(self, max_cost: int, *args) -> Tuple[int, Program]:
        return self._run(max_cost, 0, *args)

    def _run(self, max_cost: int, flags, *args) -> Tuple[int, Program]:
        # when multiple arguments are passed, concatenate them into a serialized
        # buffer. Some arguments may already be in serialized form (e.g.
        # SerializedProgram) so we don't want to de-serialize those just to
        # serialize them back again. This is handled by _serialize()
        serialized_args = b""
        if len(args) > 1:
            # when we have more than one argument, serialize them into a list
            for a in args:
                serialized_args += b"\xff"
                serialized_args += _serialize(a)
            serialized_args += b"\x80"
        else:
            serialized_args += _serialize(args[0])

        # TODO: move this ugly magic into `clvm` "dialects"
        native_opcode_names_by_opcode = dict(
            ("op_%s" % OP_REWRITE.get(k, k), op) for op, k in KEYWORD_FROM_ATOM.items() if k not in "qa."
        )
        cost, ret = deserialize_and_run_program2(
            self._buf,
            serialized_args,
            KEYWORD_TO_ATOM["q"][0],
            KEYWORD_TO_ATOM["a"][0],
            native_opcode_names_by_opcode,
            max_cost,
            flags,
        )
        return cost, Program.to(ret)

NIL = Program.from_bytes(b"\x80")


import identity_server.logic.session.login_session.logged_in_state as lst
import identity_server.logic.session.login_session.waiting_for_permission as wfp

from mongodb.Application import Application
from mongodb.ApplicationAccount import ApplicationAccount
from django.http.response import HttpResponse
import identity_server.logic.session.login_session.login_session_context as ctx
from typing import Type, Union
from django.http.request import HttpRequest
import identity_server.logic.session.session as ssn

class InitialLoginState(ssn.SessionState):
    """
    Session was not started.
    Checks is request is valid, returns login page in case if yes and bad request otherwise.
    """

    def required_request_params(self):
        return [
            'callback_url',
            'client_id'
        ]

    def route(self, request: HttpRequest) -> Union[Type[ssn.SessionState], None]:
        assert isinstance(self.session_context, ctx.LoginSessionContext)
        data = self._get_request_data(request)
        client_id = data['client_id']
        app = self.get_authorized_app(client_id)
        if app:
            self.session_context.authorized_clients[client_id] = app[0].permissions
            return lst.LoggedIn
        return super().route(request)

    def process_request(self, request: HttpRequest, **kwargs) -> HttpResponse:
        assert isinstance(self.session_context, ctx.LoginSessionContext)
        data = self._get_request_data(request)
        client_id = data['client_id']
        scope, app_name = self._get_app_info(client_id)
        is_logged_in = self.is_user_logged_in()
        self.session_context.assign(
            {'scope': scope, 'callback_url': data['callback_url'], 'client_id': client_id, 'app': app_name})

        self.set_session_state(wfp.WaitingForPermissions)
        return self.render_html(request, 'login_page.html', context={'scope': scope, 'app': app_name, 'clientId': client_id,  'is_logged_in': is_logged_in})

    def is_user_logged_in(self):
        assert isinstance(self.session_context, ctx.LoginSessionContext)
        return self.session_context.user_id != ''

    def get_authorized_app(self, client_id):
        assert isinstance(self.session_context, ctx.LoginSessionContext)
        user_id = self.session_context.user_id
        if user_id:
            authorized_app = ApplicationAccount.objects.filter(
                worker_id=user_id, client_id=client_id)
            return authorized_app

    def _get_app_info(self, client_id) -> Application:
        """
        Makes request to the database to get actual application name associated with given client id
        """
        app = Application.objects.filter(client_id=client_id).first()
        return app.permissions, app.name


from django.db import models
from datetime import datetime
from multiselectfield import MultiSelectField
from realtors.models import Realtor
from areaprops.models import Area

# Create your models here

# Choices for amenities
amenities_choices = (
    ('security','security'),
    ('gymnasium','gymnasium'),
    ('waste disposal','waste disposal'),
    ('reserved parking','reserved_parking'),
    ('lift','lift'),
    ('club house','club house'),
    ('shopping center','shopping center'),
    ('rain water harvesting','rain water harvesting'),
    ('water plant','water plant'),
    ('landscape garden','landscape garden'),
    ('kids play area','kids play area'),
    ('cctv','cctv'),
    ('cycle track','cycle track')
)

# Type of property
type_of_property = (
    ("1/2/3 BHK APARTMENT","1/2/3 BHK APARTMENT"),
    ("1/2 BHK APARTMENT","1/2 BHK APARTMENT"),
    ("1 BHK APARTMENT","1 BHK APARTMENT"),
    ("2 BHK APARTMENT","2 BHK APARTMENT"),
    ("3 BHK APARTMENT","3 BHK APARTMENT"),
    ("3 BHK DUPLEX","3 BHK DUPLEX"),
    ("2 BHK DUPLEX","2 BHK DUPLEX"),
    ("VILLA","VILLA"),
    ("BUNGALOW","BUNGALOW"),
    ("PLOT","PLOT"),
    ("PENTHOUSE","PENTHOUSE")
)

# Create your models here.
class Listing(models.Model):
    realtor = models.ForeignKey(Realtor, on_delete=models.DO_NOTHING)
    title = models.CharField(max_length=200)
    builder = models.CharField(max_length=200)
    rera_id = models.CharField(max_length=200)
    project_id = models.CharField(max_length=200)
    address = models.CharField(max_length=200)
    area = models.ForeignKey(Area, on_delete=models.DO_NOTHING)
    city = models.CharField(max_length=30,default='bhopal')
    state = models.CharField(max_length=30,default='MP')
    zipcode = models.CharField(max_length=20)
    description = models.TextField(blank=True)
    amenities = MultiSelectField(choices=amenities_choices)
    price_start = models.IntegerField()
    price_end = models.IntegerField()
    area_start = models.IntegerField()
    area_end = models.IntegerField()
    property_type = models.CharField(max_length=30,choices=type_of_property)
    possesion = models.CharField(max_length=20)
    photo_main = models.ImageField(upload_to='photos/%Y/%m/%d/')
    photo_1 = models.ImageField(upload_to='photos/%Y/%m/%d/', blank=True)
    photo_2 = models.ImageField(upload_to='photos/%Y/%m/%d/', blank=True)
    photo_3 = models.ImageField(upload_to='photos/%Y/%m/%d/', blank=True)
    photo_4 = models.ImageField(upload_to='photos/%Y/%m/%d/', blank=True)
    photo_5 = models.ImageField(upload_to='photos/%Y/%m/%d/', blank=True)
    photo_6 = models.ImageField(upload_to='photos/%Y/%m/%d/', blank=True)
    is_published = models.BooleanField(default=True)
    list_date = models.DateTimeField(default=datetime.now, blank=True)

    def __str__(self):
        return self.title


from scipy.misc import comb

def exp(p, n):
    total = 0.0
    for k in range(n+1):
        total += comb(n, k, exact=False) * p**k * (1-p) ** (n-k)

    return total

def main():
    for p in [0.3, 0.75, 0.8, 1.0, 0.0, 0.5]:
        for n in range(1, 20):
            print('Checking n=%d, p=%f' % (n, p))
            print('Result: %f' % (exp(p, n)))

if __name__ == '__main__':
    main()


from pydantic import BaseModel

class SourceURL(BaseModel):
    '''Source URL schema'''
    source_url: str

    class Config:
        orm_mode = True

class URLInfo(SourceURL):
    '''URL Information schema'''
    short_url_key: str
    short_url: str


import unittest

from onnx import defs, helper
from onnx.onnx_pb2 import NodeProto

class TestRelu(unittest.TestCase):

    def test_relu(self):
        self.assertTrue(defs.has('Relu'))
        node_def = helper.make_node(
            'Relu', ['X'], ['Y'])

if __name__ == '__main__':
    unittest.main()


'''
candidate generation: writes a pickle file of candidates

'''

import sys
import nltk
import numpy as np

from ncbi_normalization import load, sample
from ncbi_normalization.parse_MEDIC_dictionary import concept_obj

from normalize import dump_data, load_data, load_mentions

from gensim.models import KeyedVectors

def prepare_embedding_vocab(filename, binary = True, limit = 1000000):
    '''filename: '~/disease-normalization/data/embeddings/wvec_50_haodi-li-et-al.bin'
       1. Use gensim for reading in embedding model
       2. Sort based on the index to make sure that they are in the correct order
       3. Normalize the vectors
       4. Build vocabulary mappings, zero for padding
       5. Create an inverse dictionary
    '''
    vector_model = KeyedVectors.load_word2vec_format(filename, binary = binary, limit = limit)
    #vector_model=KeyedVectors.load_word2vec_format(config['embedding']['emb_file'], binary=True, limit=50000)
    words = [k for k,v in sorted(vector_model.vocab.items(),key = lambda x:x[1].index)]
    vector_model.init_sims(replace = True)
    vocabulary={"<SPECIAL>": 0, "<OOV>": 1}
    for word in words:
        vocabulary.setdefault(word, len(vocabulary))
    inversed_vocabulary={value:key for key, value in vocabulary.items()}
    return vector_model, vocabulary, inversed_vocabulary

def load_pretrained_word_embeddings(vocab,embedding_model):
    """vocab: vocabulary from data vectorizer
       embedding_model: model loaded with gensim"""
    pretrained_embeddings = np.random.uniform(low=-0.05, high=0.05, size=(len(vocab)-1,embedding_model.vectors.shape[1]))
    pretrained_embeddings = np.vstack((np.zeros(shape=(1,embedding_model.vectors.shape[1])), pretrained_embeddings))
    found=0
    for word,idx in vocab.items():
        if word in embedding_model.vocab:
            pretrained_embeddings[idx]=embedding_model.get_vector(word)
            found+=1
    print("Found pretrained vectors for {found} words.".format(found=found))
    return pretrained_embeddings

def load_concepts(dict_file,order):
    '''
    dict_file: directory to the tsv file of MEDIC dictionary
    dictionary.loaded format:
        dictionary of entries, key = canonical id, value = named tuple in the form of
        MEDIC_ENTRY(DiseaseID='MESH:D005671', DiseaseName='Fused Teeth',
        AllDiseaseIDs=('MESH:D005671',), AllNames=('Fused Teeth', 'Teeth, Fused')
    '''
    # MEDIC dictionary
    dictionary = load.Terminology()
    dictionary.loaded = load.load(dict_file,'MEDIC')

    concept = concept_obj(dictionary,order=order)
    concept.names = [name.lower() for name in concept.names]

    return concept, dictionary

def span_to_sum_of_w2v(spans,vocabulary,pretrained):
    '''
    represent all spans by sum of w2v
    '''
    embeddings = []
    for span in spans:
        tokenized = nltk.word_tokenize(span.lower())
        index = [vocabulary.get(token,1) for token in tokenized]
        #emb = np.mean(np.array([pretrained[i] for i in index]), axis=0)
        emb = np.sum(np.array([pretrained[i] for i in index]), axis=0)
        embeddings.append(emb)
    embeddings = np.array(embeddings)
    return embeddings

def cosine_similarity_candidates(mention_spans,concept_spans,emb_path,n_cossim):
    '''
    yields list of list of candidates

    n_cossim = number of candidates for each mention
    '''
    # prepare embeddings
    vector_model, vocabulary, inversed_vocabulary = prepare_embedding_vocab(emb_path, binary = True)
    pretrained = load_pretrained_word_embeddings(vocabulary, vector_model)

    # vector representations
    mention_embeddings = span_to_sum_of_w2v(mention_spans,vocabulary,pretrained)
    concept_embeddings = span_to_sum_of_w2v(concept_spans,vocabulary,pretrained)

    from sklearn.preprocessing import normalize
    concept_embeddings = normalize(concept_embeddings)
    mention_embeddings = normalize(mention_embeddings)

    dot_product_matrix = np.dot(mention_embeddings,np.transpose(concept_embeddings))
    dot_product_matrix = dot_product_matrix.tolist()
    candidate_indices = [np.argpartition(np.array(mention_candidates),-n_cossim)[-n_cossim:].tolist() for mention_candidates in dot_product_matrix]

    return candidate_indices

def jaccard_distance_candidates(mention_spans,concept_spans,n_jaccard):
    candidate_indices = []
    for mention in mention_spans:
        distances = [nltk.jaccard_distance(set(mention),set(concept)) for concept in concept_spans]
        indices = np.argpartition(np.array(distances),-n_jaccard)[-n_jaccard:].tolist()
        candidate_indices.append(indices)
    return candidate_indices

if __name__ == "__main__":
    '''
    1. prepare concept spans & mention spans
    2. get the candidates based on cosine similarity
    3. get the candidates based on Jaccard distance
    4. prepare (start, end, span), gold standard
    '''

    dict_file = 'data/CTD_diseases.tsv'
    dev_file = 'data/NCBIdevelopset_corpus.txt'
    emb_path = 'data/wvec_50_haodi-li-et-al.bin'
    n_cossim = sys.argv[1]
    n_jaccard = sys.argv[2]
    save_to = 'data/selected_max200.pickle'

    # (1)
    # concepts
    [potato0,potato1,concept_order,potato2,potato3,potato4] = load_data('data/sampled_dev_set.pickle')
    del potato0, potato1, potato2, potato3, potato4
    concept, dictionary = load_concepts(dict_file,concept_order)

    # mentions
    corpus_dev = load_mentions(dev_file,'dev corpus')

    # (2)
    cossim_candidate_indices = cosine_similarity_candidates(corpus_dev.names,concept.names,emb_path,n_cossim)

    # (3)
    jaccard_candidate_indices = jaccard_distance_candidates(corpus_dev.names,concept.names,n_jaccard)

    # (4)
    assert len(cossim_candidate_indices)==len(jaccard_candidate_indices)
    candidates = []
    for cossim,jaccard in zip(cossim_candidate_indices,jaccard_candidate_indices):
        mention_candidates = sorted(list(set(cossim+jaccard)))
        candidates.append(mention_candidates)

    positives_training, positives_dev, positives_dev_truncated = load_data('data/gitig_positive_indices.pickle')
    del positives_training, positives_dev_truncated
    positives_dev = sample.prepare_positives(positives_dev,nltk.word_tokenize,vocabulary)

    can_val_data = sample.NewDataSet('dev corpus')
    can_val_data.y = []
    can_val_data.mentions = []
    start = 0
    for cans, poss, span in zip(candidates,positives_dev,corpus_dev.names):
        end = start + len(cans)
        (chosen_idx, idces), e_token_indices = poss
        can_val_data.y.extend([1 if can in idces else 0 for can in cans])
        can_val_data.mentions.append((start,end,span))
        start = end

    assert len(can_val_data.mentions)==len(candidates)

    data = [candidates, can_val_data.mentions, can_val_data.y]
    dump_data(save_to,data)


from typing import List

class Solution:
    def minStartValue1(self, nums: List[int]) -> int:
        n = len(nums)
        m = 100
        left = 1
        right = m * n + 1
        while left < right:
            middle = (left + right) // 2
            total = middle
            is_valid = True
            for num in nums:
                total += num
                if total < 1:
                    is_valid = False
                    break
            if is_valid:
                right = middle
            else:
                left = middle + 1
        return left

    def minStartValue2(self, nums: List[int]) -> int:
        min_val = 0
        total = 0
        for num in nums:
            total += num
            min_val = min(min_val, total)
        return -min_val + 1

solution = Solution()
nums = [-3,2,-3,4,2]
assert solution.minStartValue1(nums) == 5, "Should be 5"
assert solution.minStartValue2(nums) == 5, "Should be 5"

leetcode/1413. Minimum Value to Get Positive Step by Step Sum/index.py


import sys

from osgeo import gdal, osr

class GDALUtilities:
    """
    This class has the following capabilities

    1. Get raster info
    2. Read image band as an array
    3. Reproject a raster
    """

    def __init__(self, path):
        self.path = path

    def get_raster_info(self):
        self.dataset = gdal.Open(self.path, gdal.GA_ReadOnly)
        print(
            "Driver: {}/{}\n".format(
                self.dataset.GetDriver().ShortName, self.dataset.GetDriver().LongName
            )
        )
        print(
            "Size is {} x {} x {}\n".format(
                self.dataset.RasterXSize, self.dataset.RasterYSize, self.dataset.RasterCount
            )
        )
        print("Projection is {}\n".format(self.dataset.GetProjection()))
        geotransform = self.dataset.GetGeoTransform()
        if geotransform:
            print("Origin = ({}, {})\n".format(geotransform[0], geotransform[3]))
            print("Pixel Size = ({}, {})\n".format(geotransform[1], geotransform[5]))
        band_count = self.dataset.RasterCount
        for i in range(1, band_count + 1):
            band = self.dataset.GetRasterBand(i)
            band_name = band.GetDescription()
            if band_name:
                print(f"Band {i} Name: {band_name}")
            else:
                print(f"Band {i} has no name.")
        dataset = None

    def read_image(self, band: int = None):
        self.dataset = gdal.Open(self.path, gdal.GA_ReadOnly)
        band = self.dataset.GetRasterBand(band)
        data = band.ReadAsArray()
        self.dataset = None
        return data

    def reproject(
    self, output_path: str = None, target_crs: str = None  # EPSG:4326
    ):
        self.dataset = gdal.Open(self.path, gdal.GA_ReadOnly)
        input_dataset = self.dataset
        input_srs = input_dataset.GetProjectionRef()
        target_srs = osr.SpatialReference()
        target_srs.SetFromUserInput(target_crs)
        output_dataset = gdal.Warp(
            output_path, input_dataset, dstSRS=target_srs.ExportToWkt()
        )
        input_dataset = None
        output_dataset = None


import math
import numpy as np
import statistics
import random
import time
import matplotlib.pyplot as plt

h = 40

limit_number_of_taken_values = 200
nb_of_initial_values = 100

nb_of_Dthet = 100
Dthets = [(i * 1 / nb_of_Dthet) for i in range(nb_of_Dthet)]  # thet step for ARL function

# sigs = [(0.5 + i/nb_of_sensors) for i in range(nb_of_sensors)]
# sigs = [2 for i in range(nb_of_sensors)]
sigs = [0.1, 0.5, 1.5]
nb_of_sensors = len(sigs)

def time_before_detection_step_signal(sigs, Dthet, nb_of_iteration, probas=[1] * len(sigs), h=h):
    n= len(sigs)
    nb_of_values = []
    for p in range(nb_of_iteration):
        #random.shuffle(sigs)
        X_bar = []  # somme y_i -mu_0 / sigma i
        nb_of_initial_values =random.randint(200, 200 + n )
        for i in range(nb_of_initial_values):
            sig = sigs[i % n]
            p = random.random()
            if p<probas[i % n]:
                x = np.random.normal(0, sig)
                m = len(X_bar)
                for j in range(m):
                    X_bar[j] = X_bar[j] + x / sig
                X_bar.append(x)
                # print(reception_error)
                # time.sleep(1)
        detected = False
        i = nb_of_initial_values
        while detected is False:
            sig = sigs[i % n]
            p = random.random()
            if p < probas[i % n]:
                x = np.random.normal(Dthet, sig)
                """m = len(X_bar)
                if m >= limit_number_of_taken_values:
                    X_bar = X_bar[1:]
                    m -= 1"""
                for j in range(m):
                    X_bar[j] = X_bar[j] + x / sig
                    # print((reception_error[j] * (n-j))**2)
                X_bar.append(x)
                for j in range(m + 1):
                    if (abs(X_bar[j]) / math.sqrt(m - j + 1) > h):
                        detected = True
                        # print(X_bar)
                        # print(j)
            i += 1
        nb_of_values.append(i - nb_of_initial_values)
    return statistics.mean(nb_of_values), statistics.stdev(nb_of_values), nb_of_values

def necessary_nb_of_value_GSC(sig, Dthet, h):
    return math.pow(h * sig / Dthet, 2)

def average_nb_of_necessary_values_before_detection(sig, Dthet, nb_of_iteration):
    nb_of_values = []
    for p in range(nb_of_iteration):
        i = 1
        X_bar = [np.random.normal(0, sig)]
        mean = statistics.mean(X_bar)
        while (mean + (h * sig / math.sqrt(i)) > Dthet):
            X_bar.append(np.random.normal(0, sig))
            mean = statistics.mean(X_bar)
            i += 1
        nb_of_values.append(i)
    return statistics.mean(nb_of_values)
"""

def plot_theoritical_ARL():
    nb_of_Dthet = 100
    Dthets = [(0.5 + (i * 2.5 / nb_of_Dthet)) for i in range(1, nb_of_Dthet)]
    mean = []
    for Dthet in Dthets:
        mean.append(necessary_nb_of_value_GSC(sig, Dthet))
    stri = ''
    for i in range(len(Dthets)):
        stri += '(' + str(Dthets[i]) + ',' + str(mean[i]) + ')'
    print(stri)

def plot_ARL():
    sig = 1
    std = []
    mean = []
    expected = []
    h = 10
    nb_of_iteration = 1000
    Dthet = 0.8
    pas = 0.1
    Dthets = []
    while Dthet < 2:
        Dthets.append(Dthet)
        a, b = time_before_detection_step_signal([sig], Dthet, int(nb_of_iteration), h)
        mean.append(a)
        std.append(2.567 * b / math.sqrt(nb_of_iteration))
        expected.append(necessary_nb_of_value_GSC(sig, Dthet, h))
        # print("ok")
        Dthet += pas
        pas *= 1.1
        nb_of_iteration *= 0.9
        stri = ''
        stri += '(' + str(Dthets[-1]) + ',' + str((mean[-1] - expected[-1]) * 100 / mean[-1]) + ') +- (0,)'
        print(stri)
    stri = ''
    for i in range(len(Dthets)):
        stri += '(' + str(Dthets[i]) + ',' + str((mean[i] - expected[i]) * 100 / mean[i]) + ')'
    print(stri)
    stri = ''
    for i in range(len(Dthets)):
        stri += '(' + str(Dthets[i]) + ',' + str(mean[i]) + ')'
    print(stri)
    stri = ''
    for i in range(len(Dthets)):
        stri += '(' + str(Dthets[i]) + ',' + str(expected[i]) + ')'
    print(stri)

def time_before_detection_linear_signal(sig, slope, nb_of_iteration, h=h):
    nb_of_values = []
    for p in range(nb_of_iteration):
        X_bar = []
        for i in range(nb_of_initial_values):
            x = np.random.normal(0, sig)
            n = len(X_bar)
            for j in range(n):
                X_bar[j] = X_bar[j] * (n - j) / (n - j + 1) + x / (n - j + 1)
            X_bar.append(x)
        i = 0
        detected = False
        while detected is False:
            i += 1
            x = np.random.normal(slope * i, sig)
            n = len(X_bar)
            if n >= limit_number_of_taken_values:
                X_bar = X_bar[1:]
                n -= 1
            for j in range(n):
                X_bar[j] = X_bar[j] * (n - j) / (n - j + 1) + x / (n - j + 1)
            X_bar.append(x)

            for j in range(n + 1):
                if (abs(X_bar[j]) > h * sig / math.sqrt(n + 1 - j)):
                    detected = True
                    # print(X_bar)
                    # print(j)
        nb_of_values.append(i)

        nb_of_values.append(i)
    return statistics.mean(nb_of_values), statistics.stdev(nb_of_values)

def plot_LGAARL():
    std = []
    mean = []
    nb_of_iteration = 80000
    Dthet = 0.0
    pas = 0.0005
    Dthets = []
    while Dthet < 0.4:
        Dthets.append(Dthet)
        a, b = time_before_detection_linear_signal(1, Dthet, int(nb_of_iteration))
        mean.append(a)
        std.append(2.567 * b / math.sqrt(nb_of_iteration))
        # print("ok")
        Dthet += pas
        pas *= 1.1
        nb_of_iteration *= 0.9
        stri = '(' + str(Dthets[-1]) + ',' + str(mean[-1]) + ') +- (0,' + str(std[-1]) + ')'
        print(stri)
    stri = ''
    for i in range(len(Dthets)):
        stri += '(' + str(Dthets[i]) + ',' + str(mean[i]) + ') +- (0,' + str(std[i]) + ')'
    print(stri)
"""
def main_1(Dthet):
    means = []
    stds = []
    # Dthet = 1
    nb_of_iteration = 10000
    #h = 10
    sigs = [1, 1.5, 2]
    nb_of_sensors = len(sigs)
    for sig in sigs:
        mean, std,z = time_before_detection_step_signal([sig], Dthet, int(nb_of_iteration / math.sqrt(len(sigs))))
        means.append(mean)
        stds.append(std / math.sqrt(nb_of_iteration / math.sqrt(len(sigs))))
    q = 0
    for sig in sigs:
        q += math.pow(1 / sig, 2)
    mean_adapted_one_by_one = 0
    std_adapted_one_by_one = 0
    i = 0
    for sig in sigs:
        mean_adapted_one_by_one += math.pow(1 / (q * math.pow(sig, 2)), 2) * means[i]
        std_adapted_one_by_one += stds[i] ** 2 * (1 / (q * math.pow(sig, 2))) ** 2
        i += 1
    mean_adapted_one_by_one *= len(sigs)
    std_adapted_one_by_one = math.sqrt(std_adapted_one_by_one)

    std_one_by_one = 0
    mean_one_by_one = 0
    i = 0
    for sig in sigs:
        std_one_by_one += stds[i] ** 2
        mean_one_by_one += means[i] / math.pow(len(sigs), 2)
        i += 1
    std_one_by_one = math.sqrt(std_one_by_one) / nb_of_sensors
    mean_one_by_one *= len(sigs)

    q = 0
    for m in means:
        q += 1 / m
    opti = 0
    for m in means:
        opti += math.pow(1 / (q * m), 2) * m
    opti *= len(sigs)

    mean, std, z = time_before_detection_step_signal(sigs, Dthet, nb_of_iteration)
    """
    print("one by one")
    print(mean_one_by_one)
    print(2.567 * std_one_by_one)
    print("adapted one by one")
    print(mean_adapted_one_by_one)
    print(2.567 * std_adapted_one_by_one)

    print("simultaneously")
    print(mean)
    print(2.567 * std / math.sqrt(nb_of_iteration))
    """
    return mean, 2.567 * std / math.sqrt(
        nb_of_iteration), mean_one_by_one, 2.567 * std_one_by_one, mean_adapted_one_by_one, 2.567 * std_adapted_one_by_one, opti

def main_2():
    Dthets = [1 + i * 2 / 10 for i in range(0, 10)]
    mean_simul = []
    std_simul = []
    mean_one_one = []
    std_one_one = []
    mean_adapted = []
    std_adapted = []
    mean_opti = []
    for Dthet in Dthets:
        a, b, c, d, e, f, g = main_1(Dthet)
        mean_simul.append(a)
        std_simul.append(b)
        mean_one_one.append(c)
        std_one_one.append(d)
        mean_adapted.append(e)
        std_adapted.append(f)
        mean_opti.append(g)
    moyenne = 0
    for i in range(len(Dthets)):
        moyenne += abs((mean_adapted[i] - mean_opti[i]) / mean_opti[i])
    plt.plot(Dthets, mean_simul, label='S0 round robin')
    plt.plot(Dthets, mean_one_one, label='S1 un par un un')
    plt.plot(Dthets, mean_adapted, label='S2 un par un période modifiée')
    plt.plot(Dthets, mean_opti, label="S Opt optimum global pour les stratégies un par un")

    lower_boundary = []
    upper_boundary = []
    for i in range(len(Dthets)):
        lower_boundary.append(mean_simul[i] - std_simul[i])
        upper_boundary.append(mean_simul[i] + std_simul[i])
    plt.fill_between(Dthets, lower_boundary, upper_boundary, color='#D3D3D3')
    lower_boundary = []
    upper_boundary = []
    for i in range(len(Dthets)):
        lower_boundary.append(mean_one_one[i] - std_one_one[i])
        upper_boundary.append(mean_one_one[i] + std_one_one[i])
    plt.fill_between(Dthets, lower_boundary, upper_boundary, color='#D3D3D3')

    lower_boundary = []
    upper_boundary = []
    for i in range(len(Dthets)):
        lower_boundary.append(mean_adapted[i] - std_adapted[i])
        upper_boundary.append(mean_adapted[i] + std_adapted[i])
    plt.fill_between(Dthets, lower_boundary, upper_boundary, color='#D3D3D3', label='99% confiance intervalle')
    plt.legend()
    plt.xlabel("amplitude du changement à detecter")
    plt.ylabel("temps moyen avant de lever une alerte de detection de changement")
    plt.title("comparaisons de stratégies d'émission pour des problèmes de detection en utilisant la méthode GLR")
    plt.show()

def main_3():
    Dthet = 1
    sigs = [0.1, 0.5, 1.5]
    nb_of_iteration = 1000

    ##### fst approach, nested one
    sigmas = [sigs[0],sigs[1]]
    for i in range (5):
        sigmas.append(sigs[2])
    sigmas.append(sigs[1])
    for i in range (5):
        sigmas.append(sigs[2])
    sigmas.append(sigs[1])
    for i in range(5):
        sigmas.append(sigs[2])
    sigmas.append(sigs[1])
    for i in range(5):
        sigmas.append(sigs[2])
    mean = time_before_detection_step_signal(sigmas, Dthet, nb_of_iteration, h=h)
    print(mean)

    sigs_lengths = 500

    means = []
    stds = []
    for i in range(int(nb_of_iteration/100)):
        sigmas = []
        for j in range(sigs_lengths):
            p = random.random()
            if p < 0.04:
                sigmas.append(sigs[0])
            elif p < 0.2:
                sigmas.append(sigs[1])
            else:
                sigmas.append(sigs[2])
        mean = time_before_detection_step_signal(sigmas, Dthet, 100, h=h)
        means.append(mean[0])
        stds.append(mean[1])
    print(statistics.mean(means))
    print(statistics.mean(stds))

def comparison_of_different_scheduling(nb_of_first,nb_of_second,sigma_first, sigma_second, first_proba, second_proba, nb_of_cases):
    infos = []
    for i in range(nb_of_first):
        infos.append([sigma_first,first_proba])
    for i in range(nb_of_second):
        infos.append([sigma_second, second_proba])

    nb_of_iteration = 1000
    h = 40
    Dthet = 0.5
    means = []
    stds = []
    for i in range(nb_of_cases):
        random.shuffle(infos)
        sigmas = []
        probas = []
        for elt in infos:
            sigmas.append(elt[0])
            probas.append(elt[1])
        mean, std, z = time_before_detection_step_signal(sigmas, Dthet, nb_of_iteration,probas, h)
        means.append(mean)
        stds.append(std/math.sqrt(nb_of_iteration))
    means = sorted(means)
    n = len(means)
    to_print = ""
    tot = 0
    for elt in means:
        tot += 1/n
        to_print += "(" + str(elt) +"," + str(tot) + ') '
    print(to_print)

def comparison_of_two_opposite_schedulings(nb_of_first,nb_of_second,sigma_first, sigma_second, first_proba, second_proba):

    nb_of_iteration = 50000
    h = 40
    Dthet = 1

    #### construction of the strategy where in the first time it is always the fisrt cat, then after the second cat..
    infos = []
    for i in range(nb_of_first):
        infos.append([sigma_first, first_proba])
    for i in range(nb_of_second):
        infos.append([sigma_second, second_proba])
    sigmas = []
    probas = []
    for elt in infos:
        sigmas.append(elt[0])
        probas.append(elt[1])
    mean, std, nb_of_value_before_detection = time_before_detection_step_signal(sigmas, Dthet, nb_of_iteration, probas, h)
    print(mean)
    nb_of_value_before_detection = sorted(nb_of_value_before_detection)
    values = []
    nb_of_items = []
    values.append(nb_of_value_before_detection.pop(0))
    nb_of_items.append(1)
    for elt in nb_of_value_before_detection:
        if elt ==values[-1]:
            nb_of_items[-1] += 1
        else:
            values.append(elt)
            nb_of_items.append(1)

    n = len(nb_of_value_before_detection)
    to_print = ""
    tot = 0
    for elt in zip(values, nb_of_items):
        tot += elt[1] / n
        to_print += "(" + str(elt[0]) + "," + str(tot) + ') '
    print(to_print)

    pgcd = math.gcd(nb_of_first, nb_of_second)
    infos = []
    for i in range(int(nb_of_first/pgcd)):
        infos.append([sigma_first,first_proba])
    for i in range(int(nb_of_second/pgcd)):
        infos.append([sigma_second, second_proba])
    sigmas = []
    probas = []
    for elt in infos:
        sigmas.append(elt[0])
        probas.append(elt[1])
    mean, std, nb_of_value_before_detection = time_before_detection_step_signal(sigmas, Dthet, nb_of_iteration, probas,
                                                                           h)
    print(mean)
    nb_of_value_before_detection = sorted(nb_of_value_before_detection)
    values = []
    nb_of_items = []
    values.append(nb_of_value_before_detection.pop(0))
    nb_of_items.append(1)
    for elt in nb_of_value_before_detection:
        if elt == values[-1]:
            nb_of_items[-1] += 1
        else:
            values.append(elt)
            nb_of_items.append(1)

    n = len(nb_of_value_before_detection)
    to_print = ""
    tot = 0
    for elt in zip(values, nb_of_items):
        tot += elt[1] / n
        to_print += "(" + str(elt[0]) + "," + str(tot) + ') '
    print(to_print)

def plot_CDF_of_one_random_solution(nb_of_first,nb_of_second,sigma_first, sigma_second, first_proba, second_proba):
    Dthet = 0.5
    nb_of_iteration = 10000
    h = 40
    infos = []
    for i in range(nb_of_first):
        infos.append([sigma_first, first_proba])
    for i in range(nb_of_second):
        infos.append([sigma_second, second_proba])
    random.shuffle(infos)
    sigmas = []
    probas = []
    for elt in infos:
        sigmas.append(elt[0])
        probas.append(elt[1])
    mean, std, nb_of_value_before_detection = time_before_detection_step_signal(sigmas, Dthet, nb_of_iteration, probas, h)
    print(mean)
    nb_of_value_before_detection = sorted(nb_of_value_before_detection)
    values = []
    nb_of_items = []
    values.append(nb_of_value_before_detection.pop(0))
    nb_of_items.append(1)
    for elt in nb_of_value_before_detection:
        if elt == values[-1]:
            nb_of_items[-1] += 1
        else:
            values.append(elt)
            nb_of_items.append(1)

    n = len(nb_of_value_before_detection)
    to_print = ""
    tot = 0
    for elt in zip(values, nb_of_items):
        tot += elt[1] / n
        to_print += "(" + str(elt[0]) + "," + str(tot) + ') '
    print(to_print)

def test():

    values = []
    for i in range(100000):
        values.append(np.random.normal(0,0.1)/0.1)
    values = sorted(values)
    plt.plot(values)
    plt.show()
    values = []
    for i in range(100000):
        values.append(np.random.normal(0, 1))
    values = sorted(values)
    plt.plot(values)
    plt.show()

def function_of_the_performance_according_to_the_error_noise():
    Dthet = 1
    nb_of_iteration = 10000
    h = 40
    sigs = [i/10 + 0.1 for i in range(20)]
    perfs = []
    for sig in sigs:
        mean, std, values = time_before_detection_step_signal([sig], Dthet, nb_of_iteration, probas=[1] * len(sigs), h=h)
        perfs.append(mean)
    plt.plot(sigs,perfs)
    plt.show()

if __name__ == "__main__":
    # quantify_false_positives(sigs)

    # for sig in sigs:
    #    a, b = time_before_detection_step_signal(sig, 3, 10000, h=10)
    #    print("########")
    #    print(sig)
    #    print(a)

    # plot_LGAARL()
    #main_3()
    """nb_of_first = 50
    nb_of_second = 50
    sigma_first = 0.1
    sigma_second = 0.1
    first_proba = 1
    second_proba = 1

    nb_of_cases = 2

    comparison_of_different_scheduling(nb_of_first, nb_of_second, sigma_first, sigma_second, first_proba, second_proba, nb_of_cases)
    """
    function_of_the_performance_according_to_the_error_noise()
    #comparison_of_two_opposite_schedulings(nb_of_first, nb_of_second, sigma_first, sigma_second, first_proba, second_proba)
    #plot_CDF_of_one_random_solution(nb_of_first, nb_of_second, sigma_first, sigma_second, first_proba, second_proba)


from itertools import chain

from . import builder
from .. import options as opts, safe_str, shell
from .common import Builder, choose_builder, SimpleBuildCommand
from ..file_types import HeaderFile, SourceFile
from ..iterutils import iterate
from ..languages import known_langs
from ..path import Path
from ..versioning import detect_version

# Set the source language to C++, since we want to be able to use the C++
# language definition to infer whether a file passed to `moc` is a source or
# header file based on its extension.
with known_langs.make('qtmoc', src_lang='c++') as x:
    x.vars(compiler='MOC', flags='MOCFLAGS')

with known_langs.make('qrc') as x:
    x.vars(compiler='RCC', flags='RCCFLAGS')
    x.exts(source=['.qrc'])

with known_langs.make('qtui') as x:
    x.vars(compiler='UIC', flags='UICFLAGS')
    x.exts(source=['.ui'])

@builder('qtmoc')
def moc_builder(env):
    return choose_builder(env, known_langs['qtmoc'], (MocBuilder,),
                          default_candidates=['moc'])

class MocBuilder(Builder):
    def __init__(self, env, langinfo, command, found, version_output):
        super().__init__(langinfo.name, *self._parse_brand(version_output))

        name = langinfo.var('compiler').lower()
        mocflags_name = langinfo.var('flags').lower()
        mocflags = shell.split(env.getvar(langinfo.var('flags'), ''))

        self.transpiler = MocCompiler(
            self, env, command=(name, command, found),
            flags=(mocflags_name, mocflags)
        )

    @staticmethod
    def _parse_brand(version_output):
        if 'moc' in version_output:
            return 'qt', detect_version(version_output)
        return 'unknown', None

    @staticmethod
    def check_command(env, command):
        return env.execute(command + ['--version'], stdout=shell.Mode.pipe,
                           stderr=shell.Mode.devnull)

class MocCompiler(SimpleBuildCommand):
    @property
    def deps_flavor(self):
        return None

    def _call(self, cmd, input, output, flags=None):
        return list(chain(
            cmd, iterate(flags), [input, '-o', output]
        ))

    def default_name(self, input, step):
        if isinstance(input, SourceFile):
            return input.path.stripext('.moc').suffix
        base, leaf = input.path.stripext(
            known_langs['c++'].default_ext('source')
        ).splitleaf()
        return base.append('moc_' + leaf).suffix

    def output_file(self, name, step):
        return SourceFile(Path(name), 'c++')

    def flags(self, options, global_options=None, output=None, mode='normal'):
        flags = []
        for i in options:
            if isinstance(i, opts.include_dir):
                flags.append('-I' + i.directory.path)
            elif isinstance(i, opts.define):
                if i.value:
                    flags.append('-D' + i.name + '=' + i.value)
                else:
                    flags.append('-D' + i.name)
            elif isinstance(i, opts.warning):
                for j in i.value:
                    if j == opts.WarningValue.disable:
                        flags.append('--no-warnings')
                    else:
                        raise ValueError('unsupported warning level {!r}'
                                         .format(j))
            elif isinstance(i, safe_str.stringy_types):
                flags.append(i)
            else:
                raise TypeError('unknown option type {!r}'.format(type(i)))
        return flags

@builder('qrc')
def qrc_builder(env):
    return choose_builder(env, known_langs['qrc'], (RccBuilder,),
                          default_candidates=['rcc'])

class RccBuilder(Builder):
    def __init__(self, env, langinfo, command, found, version_output):
        super().__init__(langinfo.name, *self._parse_brand(version_output))

        name = langinfo.var('compiler').lower()
        rccflags_name = langinfo.var('flags').lower()
        rccflags = shell.split(env.getvar(langinfo.var('flags'), ''))

        self.transpiler = RccCompiler(
            self, env, command=(name, command, found),
            flags=(rccflags_name, rccflags)
        )

    @staticmethod
    def _parse_brand(version_output):
        if 'rcc' in version_output:
            return 'qt', detect_version(version_output)
        return 'unknown', None

    @staticmethod
    def check_command(env, command):
        return env.execute(command + ['--version'], stdout=shell.Mode.pipe,
                           stderr=shell.Mode.devnull)

class RccCompiler(SimpleBuildCommand):
    @property
    def deps_flavor(self):
        return 'gcc'

    def _call(self, cmd, input, output, deps=None, flags=None):
        result = list(chain(cmd, iterate(flags), [input, '-o', output]))
        if deps:
            return self.env.tool('rccdep')(result, deps)
        return result

    def default_name(self, input, step):
        return input.path.stripext(
            known_langs['c++'].default_ext('source')
        ).suffix

    def output_file(self, name, step):
        return SourceFile(Path(name), 'c++')

    def flags(self, options, global_options=None, output=None, mode='normal'):
        flags = []
        for i in options:
            if isinstance(i, safe_str.stringy_types):
                flags.append(i)
            else:
                raise TypeError('unknown option type {!r}'.format(type(i)))
        return flags

@builder('qtui')
def qtui_builder(env):
    return choose_builder(env, known_langs['qtui'], (UicBuilder,),
                          default_candidates=['uic'])

class UicBuilder(Builder):
    def __init__(self, env, langinfo, command, found, version_output):
        super().__init__(langinfo.name, *self._parse_brand(version_output))

        name = langinfo.var('compiler').lower()
        uicflags_name = langinfo.var('flags').lower()
        uicflags = shell.split(env.getvar(langinfo.var('flags'), ''))

        self.transpiler = UicCompiler(
            self, env, command=(name, command, found),
            flags=(uicflags_name, uicflags)
        )

    @staticmethod
    def _parse_brand(version_output):
        if 'uic' in version_output:
            return 'qt', detect_version(version_output)
        return 'unknown', None

    @staticmethod
    def check_command(env, command):
        return env.execute(command + ['--version'], stdout=shell.Mode.pipe,
                           stderr=shell.Mode.devnull)

class UicCompiler(SimpleBuildCommand):
    @property
    def deps_flavor(self):
        return None

    def _call(self, cmd, input, output, flags=None):
        return list(chain(
            cmd, iterate(flags), [input, '-o', output]
        ))

    def default_name(self, input, step):
        base, leaf = input.path.stripext('.h').splitleaf()
        return base.append('ui_' + leaf).suffix

    def output_file(self, name, step):
        return HeaderFile(Path(name), 'c++')

    def flags(self, options, global_options=None, output=None, mode='normal'):
        flags = []
        for i in options:
            if isinstance(i, safe_str.stringy_types):
                flags.append(i)
            else:
                raise TypeError('unknown option type {!r}'.format(type(i)))
        return flags


import math

import matplotlib.pyplot as plt
import numpy as np
from matplotlib import gridspec
from scipy.special import factorial

from plot.plot_data import plot_matrixImage

def normalize(X):
    f_min, f_max = X.min(), X.max()
    return (X - f_min) / (f_max - f_min)

def gabor_kernel_2(frequency, sigma_x, sigma_y, theta=0, offset=0, ks=61):
    w = np.floor(ks / 2)
    y, x = np.mgrid[-w:w + 1, -w:w + 1]
    rotx = x * np.cos(theta) + y * np.sin(theta)
    roty = -x * np.sin(theta) + y * np.cos(theta)
    g = np.zeros(y.shape)
    g[:] = np.exp(-0.5 * (rotx ** 2 / sigma_x ** 2 + roty ** 2 / sigma_y ** 2))
    g /= 2 * np.pi * sigma_x * sigma_y
    g *= np.cos(2 * np.pi * frequency * rotx + offset)
    return g

def gabor_kernel_3(frequency, x_c, y_c, sigma_x, sigma_y, theta=0, offset=0, ks=61, scale=1):
    w = np.floor(ks / 2)
    y, x = np.mgrid[-w:w + 1, -w:w + 1]
    rotx = (x - x_c) * np.cos(theta) + (y - y_c) * np.sin(theta)
    roty = -(x - x_c) * np.sin(theta) + (y - y_c) * np.cos(theta)
    g = np.zeros(y.shape)
    g[:] = np.exp(-0.5 * (rotx ** 2 / sigma_x ** 2 + roty ** 2 / sigma_y ** 2))
    g /= 2 * np.pi * sigma_x * sigma_y
    g *= np.cos(2 * np.pi * frequency * rotx + offset)
    return g * scale

def poisson(k, lamb):
    """poisson pdf, parameter lamb is the fit parameter"""
    return (lamb ** k / factorial(k)) * np.exp(-lamb)

def negLogLikelihood(params, data):
    """ the negative log-Likelohood-Function"""
    lnl = - np.sum(np.log(poisson(data, params[0])))
    return lnl

# def tfm_poisson_pdf(x, mu):
#     y, J = transformation_and_jacobian(x)
#     # For numerical stability, compute exp(log(f(x)))
#     return np.exp(y * np.log(mu) - mu - gammaln(y + 1.)) * J

def plot_conv_weights(weights, model_name):
    length = weights.shape[0] * weights.shape[2]
    matrix = np.zeros([length, 0])
    for i in range(0, weights.shape[0]):
        row = np.empty([0, weights.shape[2]])
        for j in range(0, weights.shape[1]):
            row = np.concatenate((row, weights[i, j]), axis=0)
        # f_min, f_max = np.min(row), np.max(row)
        # row = (row - f_min) / (f_max - f_min)
        # row[0,0] = 0
        matrix = np.concatenate((matrix, row), axis=1)
        # matrix[0,0] = 1
    f_min, f_max = np.min(matrix), np.max(matrix)
    matrix = (matrix - f_min) / (f_max - f_min)
    plot_matrixImage(matrix, 'weights_' + model_name)

def plot_weights(weights, model_name, gs=None, name=None):
    show = False
    if gs is None:
        plt.figure(figsize=(10, 2), frameon=False)
        inner = gridspec.GridSpec(weights.shape[0], weights.shape[1], wspace=0.2, hspace=0.2)
        show = True
    else:
        inner = gridspec.GridSpecFromSubplotSpec(weights.shape[0], 8,
                                                 subplot_spec=gs, wspace=0.1, hspace=0.1)
    # gs = gridspec.GridSpec(, width_ratios=[1] * weights.shape[1],
    #                        wspace=0.5, hspace=0.5, top=0.95, bottom=0.05, left=0.1, right=0.95)
    idx = 0
    for i in range(0, weights.shape[0]):
        for j in range(0, weights.shape[1]):
            kernel1 = weights[i, j]
            ax_ = plt.subplot(inner[i, j])
            ax_.set_xticks([])
            ax_.set_yticks([])
            ax_.set_axis_off()
            ax_.imshow(kernel1, cmap='gray')
            #
            idx += 1
            if j == 0:
                ax_.set_title(name, pad=10, weight='semibold', size=16)
    if show:
        plt.tight_layout()
        plt.savefig(f'weights_{model_name}.png')
        plt.show()

def show_kernels(weights, func_name, gs=None):
    number = math.ceil(math.sqrt(weights.shape[0]))
    img = np.transpose(weights, (0, 2, 3, 1))
    idx = 0
    show = False
    if gs is None:
        plt.figure(figsize=(10, 10))
        inner = gridspec.GridSpec(1, weights.shape[0], wspace=0.2, hspace=0.2)
        show = True
    else:
        inner = gridspec.GridSpecFromSubplotSpec(1, 8,
                                                 subplot_spec=gs, wspace=0.1, hspace=0.1)

    # fig, axes = pyplot.subplots(ncols=weights.shape[0], figsize=(20, 4))
    for j in range(weights.shape[0]):  # in zip(axes, range(weights.shape[0])):
        # for i in range(number):
        ax_ = plt.subplot(inner[idx])
        ax_.set_xticks([])
        ax_.set_yticks([])
        ax_.set_axis_off()
        # ax.set_title(f'Kernel {idx}', pad=3)
        # imgs = img[range(j*8, (j*8)+number)]
        channel = img[idx]
        f_min, f_max = channel.min(), channel.max()
        channel = (channel - f_min) / (f_max - f_min)
        ax_.imshow(channel)
        if j == 0:
            ax_.set_title(func_name, pad=10, weight='bold', size=18)
        idx += 1

    if show:
        plt.tight_layout()
        plt.savefig(f'kernels_{func_name}.png')
        plt.show()

def similarity(m1, m2):
    sum = 0
    for i in range(m1.shape[0]):
        for j in range(m1.shape[1]):
            sum += np.abs(m1[i, j] - m2[i, j])
    return sum / (m1.shape[0] * m1.shape[1])