from __future__ import annotations

from asyncio import (CancelledError, Task, Queue, # pylint: disable=unused-import; typing
                     create_task, ensure_future, get_event_loop, shield, sleep)
import builtins
from datetime import datetime, timedelta, timezone
from email.message import EmailMessage
from functools import partial
from inspect import isawaitable
import json
from logging import getLogger
from pathlib import Path
import re
from smtplib import SMTP
import string
import sys
from typing import (AsyncIterator, Awaitable, Callable, Coroutine, Dict, Generic, Iterator, List,
                    Optional, Set, Sequence, Tuple, Type, TypeVar, Union, cast, overload)
from urllib.parse import SplitResult, urlparse, urlsplit

from pywebpush import WebPusher, WebPushException
from py_vapid import Vapid
from py_vapid.utils import b64urlencode
from redis import StrictRedis
from redis.exceptions import ResponseError
from requests.exceptions import RequestException
from typing_extensions import Protocol

from . import error
from .core import Device, Devices, Object, RewriteFunc, context
from .error import ValueError
from .jsonredis import (ExpectFunc, JSONRedis, JSONRedisSequence, RedisList, RedisSequence,
                        RedisSortedSet, bzpoptimed)
from .ratelimit import RateLimit, RateLimiter
from .resource import ( # pylint: disable=unused-import; typing
    AnalysisError, Analyzer, Files, HandleResourceFunc, Image, Resource, Video, handle_image,
    handle_webpage, handle_youtube)
from .util import Expect, check_email, expect_opt_type, expect_type, randstr, str_or_none
from .webapi import CommunicationError

_PUSH_TTL = 24 * 60 * 60

O = TypeVar('O', bound='Object')

class JSONifiable(Protocol):
    """Object which can be encoded to and decoded from a JSON representation."""

    def __init__(self, **kwargs: object) -> None:
        # pylint: disable=super-init-not-called; protocol
        pass

    def json(self, restricted: bool = False, include: bool = False, *,
             rewrite: RewriteFunc = None) -> Dict[str, object]:
        """Return a JSON representation of the object.

        The name of the object type is included as ``__type__``.

        By default, all attributes are included. If *restricted* is ``True``, a restricted view of
        the object is returned, i.e. attributes that should not be available to the current
        :attr:`Application.user` are excluded. If *include* is ``True``, additional fields that may
        be of interest to the caller are included. If *rewrite* is given, it is applied to all URL
        attributes.
        """

class JSONifiableWithParse(JSONifiable, Protocol):
    # pylint: disable=missing-docstring; semi-public

    @staticmethod
    def parse(data: Dict[str, object], **args: object) -> JSONifiable:
        """Parse the given JSON *data* into an object.

        Additional keyword arguments *args* are passed to the constructor.
        """

class Application:
    """Social micro web app.

    .. attribute:: user

       Current :class:`User`. ``None`` means anonymous access.

    .. attribute:: users

       Collection of all :class:`User`s.

    .. attribute:: analytics

       Analytics unit.

    .. attribute:: redis_url

       See ``--redis-url`` command line option.

    .. attribute:: email

       Sender email address to use for outgoing email. Defaults to ``bot@localhost``.

    .. attribute:: smtp_url

       See ``--smtp-url`` command line option.

    .. attribute:: render_email_auth_message

       Hook function of the form *render_email_auth_message(email, auth_request, auth)*, responsible
       for rendering an email message for the authentication request *auth_request*. *email* is the
       email address to authenticate and *auth* is the secret authentication code.

    .. attribute:: video_service_keys

       See ``--video-service-keys`` command line option.

    .. attribute:: r

       :class:`Redis` database. More precisely a :class:`JSONRedis` instance.

    .. attribute:: files

       File storage. *files_path* is the directory where files are stored (see ``--files-path``
       command line option).

    .. attribute:: analyzer

       Web resource analyzer.

    .. attribute:: rate_limiter

       Subclass API: Mechanism to limit the rate of operations per client.
    """

    def __init__(
            self, redis_url: str = '', email: str = 'bot@localhost', smtp_url: str = '',
            render_email_auth_message: Callable[[str, 'AuthRequest', str], str] = None, *,
            files_path: str = 'data', video_service_keys: Dict[str, str] = {}) -> None:
        check_email(email)
        try:
            # pylint: disable=pointless-statement; port errors are only triggered on access
            urlparse(smtp_url).port
        except builtins.ValueError as e:
            raise ValueError('smtp_url_invalid') from e

        self.redis_url = redis_url
        try:
            urlparts = urlsplit(self.redis_url)
            url = SplitResult(
                urlparts.scheme or 'redis', urlparts.netloc or 'localhost', urlparts.path,
                urlparts.query, urlparts.fragment
            ).geturl()
            self.r = JSONRedis(StrictRedis.from_url(url), self._encode, self._decode)
        except builtins.ValueError as e:
            raise ValueError('redis_url_invalid') from e
        self.files = Files(files_path)

        # pylint: disable=import-outside-toplevel; circular dependency
        from .analytics import Analytics, Referral
        self.types = {
            'User': User,
            'Device': Device,
            'Settings': Settings,
            'Activity': Activity,
            'Event': Event,
            'AuthRequest': AuthRequest,
            'Resource': Resource,
            'Image': Image,
            'Video': Video,
            'Referral': Referral
        } # type: Dict[str, Type[JSONifiable]]
        self.user = None # type: Optional[User]
        self.users = Users(self)
        self.devices = Devices(self)
        self.analytics = Analytics(app=self)

        self.email = email
        self.smtp_url = smtp_url
        self.render_email_auth_message = render_email_auth_message

        self.video_service_keys = video_service_keys
        handlers = [handle_image, handle_webpage] # type: List[HandleResourceFunc]
        if 'youtube' in self.video_service_keys:
            handlers.insert(0, handle_youtube(self.video_service_keys['youtube']))
        self.analyzer = Analyzer(handlers=handlers, files=self.files)
        self.rate_limiter = RateLimiter()

    @property
    def settings(self) -> 'Settings':
        """App :class:`Settings`."""
        return self.r.oget('Settings', default=AssertionError, expect=expect_type(Settings))

    @property
    def activity(self) -> 'Activity':
        """Global :class:`Activity` feed."""
        activity = self.r.oget('Activity', default=AssertionError, expect=expect_type(Activity))
        activity.pre = self.check_user_is_staff
        return activity

    @staticmethod
    def now() -> datetime:
        """Return the current UTC date and time, as aware object with second accuracy."""
        return datetime.now(timezone.utc).replace(microsecond=0)

    async def update(self) -> None:
        """Update the database.

        If the database is fresh, it will be initialized. If the database is already up-to-date,
        nothing will be done. It is thus safe to call :meth:`update` without knowing if an update is
        necessary or not.
        """
        Path(self.files.path).mkdir(parents=True, exist_ok=True)

        v = cast(Optional[bytes], self.r.get('micro_version'))
        # If fresh, initialize database
        if not v:
            settings = self.create_settings()
            settings.push_vapid_private_key, settings.push_vapid_public_key = (
                self._generate_push_vapid_keys()) # type: ignore
            self.r.oset(settings.id, settings)
            activity = Activity(id='Activity', app=self, subscriber_ids=[])
            self.r.oset(activity.id, activity)
            self.r.set('micro_version', 9)
            self.do_update()
            return

        v = int(v)
        r = JSONRedis[Dict[str, object]](self.r.r)
        r.caching = False

        user_updates = {}
        device_updates = {}
        users = r.omget([id.decode() for id in r.r.lrange('users', 0, -1)], default=AssertionError)
        for user in users:
            # Deprecated since 0.58.0
            if 'auth_secret' in user:
                device = Device(
                    id=f'Device:{randstr()}', app=self, auth_secret=user.pop('auth_secret'),
                    notification_status=user.pop('device_notification_status'),
                    push_subscription=user.pop('push_subscription'), user_id=user['id'])
                r.sadd('devices', device.id)
                authenticate_time = datetime.fromisoformat(cast(str, user['authenticate_time']))
                r.zadd(f"{user['id']}.devices", {device.id: -authenticate_time.timestamp()})
                r.hset('auth_secret_map', device.auth_secret, device.id)
                user_updates[cast(str, user['id'])] = user
                device_updates[device.id] = device.json()
        r.omset(user_updates)
        r.omset(device_updates)

        object_updates = {}
        for obj in self._scan_objects(r, WithContent):
            id = cast(str, obj['id'])
            resource = cast('dict[str, object] | None', obj['resource'])
            if resource:
                # Deprecated since 0.67.0
                if 'thumbnail' not in resource:
                    src = (resource if resource['__type__'] == 'Image'
                           else cast('dict[str, object] | None', resource['image']))
                    if src:
                        try:
                            thumbnail = await self.analyzer.thumbnail(cast(str, src['url']))
                        except (CommunicationError, AnalysisError):
                            thumbnail = await self.analyzer.thumbnail(
                                b'<svg xmlns="http://www.w3.org/2000/svg" />', 'image/svg+xml')
                        resource['thumbnail'] = thumbnail.json()
                    else:
                        resource['thumbnail'] = None
                    del resource['image']
                    object_updates[id] = obj

                # Deprecated since 0.69.0
                else:
                    thumbnail_data = cast('dict[str, object] | None', resource['thumbnail'])
                    if thumbnail_data and 'color' not in thumbnail_data:
                        thumbnail = await self.analyzer.thumbnail(cast(str, thumbnail_data['url']))
                        thumbnail_data['color'] = thumbnail.color
                        object_updates[id] = obj
        r.omset(object_updates)

        updates = {
            'User': len(user_updates),
            'Device': len(device_updates),
            'Object': len(object_updates),
            **self.do_update()
        }
        getLogger(__name__).info('Updated database\n%s',
                                 '\n'.join(f'{name}: {n}' for name, n in updates.items()))

    def do_update(self) -> Dict[str, int]:
        """Subclass API: Perform the database update.

        Information about the update, more precisely the number of entity updates by type, is
        returned.

        May be overridden by subclass. Called by :meth:`update`, which takes care of updating (or
        initializing) micro specific data. The default implementation does nothing.
        """
        # pylint: disable=no-self-use; part of subclass API
        return {}

    def create_user(self, data: Dict[str, object]) -> 'User':
        """Subclass API: Create a new user.

        *data* is the JSON data required for :class:`User`.

        May be overridden by subclass. Called by :meth:`login` for new users, which takes care of
        storing the object. By default, a standard :class:`User` is returned.
        """
        # pylint: disable=no-self-use; part of subclass API
        return User(**data) # type: ignore

    def create_settings(self) -> 'Settings':
        """Subclass API: Create and return the app :class:`Settings`.

        *id* must be set to ``Settings``.

        Must be overridden by subclass. Called by :meth:`update` when initializing the database.
        """
        raise NotImplementedError()

    def get_object(self, id, default=KeyError):
        """Get the :class:`Object` given by *id*.

        *default* is the value to return if no object with *id* is found. If it is an
        :exc:`Exception`, it is raised instead.
        """
        object = self.r.oget(id)
        if object is None:
            object = default
        if isinstance(object, Exception):
            raise object
        return object

    def check_user_is_staff(self) -> None:
        """Check if the current :attr:`user` is a staff member."""
        # pylint: disable=protected-access; Settings is a friend
        if not (self.user and self.user.id in self.settings._staff): # type: ignore
            raise error.PermissionError()

    def start_garbage_collect_files(self) -> 'Task[None]':
        """Start the :attr:`files` garbage collect job."""
        async def _run() -> None:
            t = self.now().replace(hour=0, minute=5, second=0)
            while True:
                t += timedelta(days=1)
                await sleep((t - self.now()).total_seconds())
                n = await self.files.garbage_collect(list(self.file_references()))
                getLogger(__name__).info('Garbage collected %d file(s)', n)
        return cast('Task[None]', get_event_loop().create_task(_run()))

    def start_empty_trash(self) -> 'Task[None]':
        """Start the empty trash job."""
        async def _empty_trash() -> None:
            loop = get_event_loop()
            while True:
                coro = cast(
                    Awaitable[Tuple[bytes, float]],
                    loop.run_in_executor(None, partial(bzpoptimed, self.r.r, 'micro_trash')))
                try:
                    id, _ = await shield(coro)
                except CancelledError:
                    # Note that we assume there is only a single consumer of micro_trash
                    self.r.r.zadd('micro_trash', {b'int': 0})
                    await coro
                    raise

                obj = self.r.oget(id.decode(), default=AssertionError,
                                  expect=expect_type(Trashable))
                # pylint: disable=broad-except; catch unhandled exceptions
                try:
                    obj.delete()
                except Exception as e:
                    t = (
                        datetime.now(timezone.utc) + Trashable.RETENTION).timestamp() # type: ignore
                    self.r.r.zadd('micro_trash', {id: t})
                    loop.call_exception_handler(
                        cast(Dict[str, object], {
                            'message': 'Unexpected exception in delete()',
                            'exception': e
                        }))
                del obj
        return cast('Task[None]', ensure_future(_empty_trash()))

    def file_references(self) -> Iterator[str]:
        """Subclass API: Iterate over all file references.

        Called by the :attr:`files` garbage collect job. The default implementation returns an empty
        iterator.
        """
        # pylint: disable=no-self-use; part of API
        return iter(())

    async def send_device_notification(self, push_subscription: str, event: Event) -> None:
        """Send an *event* notification to the device with *push_subscription*.

        If there is a communication issue, a :exc:`micro.error.CommunicationError` is raised.
        """
        try:
            push_subscription = json.loads(push_subscription)
            if not isinstance(push_subscription, dict):
                raise builtins.ValueError()
            urlparts = urlparse(push_subscription['endpoint'])
            pusher = WebPusher(push_subscription)
        except (builtins.ValueError, KeyError, WebPushException) as e:
            raise ValueError('push_subscription_invalid') from e

        # Unfortunately sign() tries to validate the email address
        email = 'bot@email.localhost' if self.email == 'bot@localhost' else self.email
        headers = Vapid.from_raw(self.settings.push_vapid_private_key.encode()).sign({
            'aud': '{}://{}'.format(urlparts.scheme, urlparts.netloc),
            'sub': 'mailto:{}'.format(email)
        })

        try:
            # Firefox does not yet support aes128gcm encoding (see
            # https://bugzilla.mozilla.org/show_bug.cgi?id=1525872)
            send = partial(pusher.send, json.dumps(event.json(restricted=True, include=True)),
                           headers=headers, ttl=_PUSH_TTL, content_encoding='aesgcm')
            response = await get_event_loop().run_in_executor(None, send)
        except RequestException as e:
            raise CommunicationError(f"{e} for POST {push_subscription['endpoint']}") from e
        if response.status_code in (404, 410):
            raise ValueError('push_subscription_invalid')
        if response.status_code != 201:
            raise CommunicationError(
                'Unexpected response status {} for POST {}'.format(response.status_code,
                                                                   push_subscription['endpoint']))

    @staticmethod
    def _encode(object: JSONifiable) -> Dict[str, object]:
        return object.json()

    def _decode(self, json: Dict[str, object]) -> Union[JSONifiable, Dict[str, object]]:
        # pylint: disable=redefined-outer-name; good name
        try:
            type = self.types[str(json['__type__'])]
        except KeyError:
            return json
        if hasattr(type, 'parse'):
            return cast(JSONifiableWithParse, type).parse(json, app=self)
        del json['__type__']
        return type(app=self, **json)

    def _scan_objects(self, r: JSONRedis[dict[str, object]],
                      cls: type[object] = None) -> Iterator[dict[str, object]]:
        for key in cast(List[bytes], r.keys('*')):
            try:
                obj = r.oget(key.decode(), default=AssertionError)
            except ResponseError:
                pass
            else:
                if ('__type__' in obj and
                        issubclass(self.types[expect_type(str)(obj['__type__'])], cls or Object)):
                    yield obj

    @staticmethod
    def _generate_push_vapid_keys():
        vapid = Vapid()
        vapid.generate_keys()
        return (b64urlencode(vapid.private_key.private_numbers().private_value.to_bytes(32, 'big')),
                b64urlencode(vapid.public_key.public_numbers().encode_point()))

class Gone:
    """See :ref:`Gone`."""

    def json(self, restricted: bool = False, include: bool = False, *,
             rewrite: RewriteFunc = None) -> Dict[str, object]:
        """See :meth:`JSONifiable.json`."""
        # pylint: disable=unused-argument; part of API
        return {'__type__': type(self).__name__}

class Editable:
    """:class:`Object` that can be edited."""

    id = None # type: str
    app = None # type: Application
    trashed = None # type: bool

    def __init__(self, authors: List[str],
                 activity: Union['Activity', Callable[[], 'Activity']] = None) -> None:
        self._authors = authors
        self.__activity = activity

    @property
    def authors(self) -> List['User']:
        # pylint: disable=missing-docstring; already documented
        return self.app.r.omget(self._authors, default=AssertionError, expect=expect_type(User))

    async def edit(self, **attrs: object) -> None:
        """See :http:post:`/api/(object-url)`."""
        if not self.app.user:
            raise error.PermissionError()
        if isinstance(self, Trashable) and self.trashed:
            raise ValueError('object_trashed')

        coro = self.do_edit(**attrs)
        if isawaitable(coro):
            await cast(Awaitable[None], coro)
        if not self.app.user.id in self._authors:
            self._authors.append(self.app.user.id)
        self.app.r.oset(self.id, self)

        if self.__activity is not None:
            activity = self.__activity() if callable(self.__activity) else self.__activity
            activity.publish(Event.create('editable-edit', self, app=self.app)) # type: ignore

    def do_edit(self, **attrs: object) -> Optional[Awaitable[None]]:
        """Subclass API: Perform the edit operation.

        More precisely, validate and then set the given *attrs*.

        Must be overridden by host. Called by :meth:`edit`, which takes care of basic permission
        checking, managing *authors* and storing the updated object in the database.
        """
        raise NotImplementedError()

    def json(self, restricted: bool = False, include: bool = False, *,
             rewrite: RewriteFunc = None) -> Dict[str, object]:
        """Subclass API: Return a JSON object representation of the editable part of the object."""
        return {
            'authors': [a.json(restricted=restricted, rewrite=rewrite) for a in self.authors]
                       if include else self._authors
        }

class Trashable:
    """Mixin for :class:`Object` which can be trashed and restored.

    .. attribute:: RETENTION

       Duration after a trashed object will permanently be deleted.
    """

    RETENTION = timedelta(days=7)

    id = None # type: str
    app = None # type: Application

    def __init__(self, trashed: bool,
                 activity: Union['Activity', Callable[[], 'Activity']] = None) -> None:
        self.trashed = trashed
        self.__activity = activity

    def delete(self) -> None:
        """Subclass API: Permanently delete the object."""
        raise NotImplementedError()

    def trash(self) -> None:
        """See :http:post:`/api/(object-url)/trash`."""
        if not self.app.user:
            raise error.PermissionError()
        if self.trashed:
            return

        self.trashed = True
        self.app.r.oset(self.id, self)
        self.app.r.r.zadd(
            'micro_trash',
            {self.id.encode(): (datetime.now(timezone.utc) + Trashable.RETENTION).timestamp()})
        if self.__activity is not None:
            activity = self.__activity() if callable(self.__activity) else self.__activity
            activity.publish(Event.create('trashable-trash', cast(Object, self), app=self.app))

    def restore(self) -> None:
        """See :http:post:`/api/(object-url)/restore`."""
        if not self.app.user:
            raise error.PermissionError()
        if not self.trashed:
            return

        self.trashed = False
        self.app.r.oset(self.id, self)
        self.app.r.r.zrem('micro_trash', self.id.encode())
        if self.__activity is not None:
            activity = self.__activity() if callable(self.__activity) else self.__activity
            activity.publish(Event.create('trashable-restore', cast(Object, self), app=self.app))

    def json(self, restricted: bool = False, include: bool = False, *,
             rewrite: RewriteFunc = None) -> Dict[str, object]:
        """Subclass API: Return a JSON representation of the trashable part of the object."""
        # pylint: disable=unused-argument; part of subclass API
        return {'trashed': self.trashed}

class WithContent:
    """:class:`Editable` :class:`Object` with content."""

    app = None # type: Application

    @staticmethod
    async def process_attrs(attrs: Dict[str, object], *, app: Application) -> Dict[str, object]:
        """Pre-Process the given attributes *attrs* for editing."""
        if 'text' in attrs:
            text = attrs['text']
            if text is not None:
                attrs['text'] = str_or_none(expect_type(str)(text))
        if 'resource' in attrs:
            url = attrs['resource']
            if url is not None:
                attrs['resource'] = await app.analyzer.analyze(expect_type(str)(url))
        return attrs

    def __init__(self, *, text: str = None, resource: Resource = None) -> None:
        self.text = text
        self.resource = resource

    async def pre_edit(self, attrs: Dict[str, object]) -> Dict[str, object]:
        """Prepare the edit operation.

        More precisely, validate and pre-process the given *attrs*.
        """
        if 'resource' in attrs and self.resource and attrs['resource'] == self.resource.url:
            del attrs['resource']
        return await self.process_attrs(attrs, app=self.app)

    def do_edit(self, **attrs: object) -> Optional[Awaitable[None]]: # type: ignore
        """See :meth:`Editable.do_edit`."""
        if 'text' in attrs:
            self.text = expect_opt_type(str)(attrs['text'])
        if 'resource' in attrs:
            self.resource = expect_opt_type(Resource)(attrs['resource'])

    def json(self, restricted: bool = False, include: bool = False, *,
             rewrite: RewriteFunc = None) -> Dict[str, object]:
        """See :meth:`JSONifiable.json`."""
        # pylint: disable=unused-argument; part of API
        return {
            'text': self.text,
            'resource': self.resource.json(rewrite=rewrite) if self.resource else None
        }

class Collection(Generic[O], Sequence[O]):
    """See :ref:`Collection`.

    .. attribute:: ids

       Redis sequence tracking the (IDs of) objects that the collection contains.

    .. attribute:: check

       Function of the form *check(key)*, which is called before an object is retrieved via *key*.
       May be ``None``.

    .. attribute:: expect

       Function narrowing the type of a retrieved object.

    .. attribute:: app

       Context application.

    .. describe:: c[key]

       *key* may also be a string, in which case the object with the given ID is retrieved.

       Redis operations: ``RedisSequence[k] + n``, where n is the number of retrieved items, or
       ``i in RedisSequence + 1`` if *key* is a string.

    .. describe:: item in c

       *item* may also be a string, in which case the membership of the object with the given ID is
       tested.
    """

    def __init__(
            self, ids: RedisSequence, *, check: Callable[[Union[int, slice, str]], None] = None,
            expect: ExpectFunc[O] = cast(ExpectFunc[O], expect_type(Object)),
            app: Application) -> None:
        self.ids = ids
        self.check = check
        self.expect = expect
        self.app = app

    def index(self, value: Union[O, str], start: int = 0, stop: int = sys.maxsize) -> int:
        """See :meth:`Sequence.index`.

        *x* may also be a string, in which case the index of the object with the given ID is
        returned.
        """
        if isinstance(value, Object):
            return self.index(value.id, start, stop)
        return self.ids.index(cast(str, value).encode(), start, stop)

    def __len__(self) -> int:
        return len(self.ids)

    @overload
    def __getitem__(self, key: Union[int, str]) -> O:
        # pylint: disable=function-redefined; overload
        pass
    @overload
    def __getitem__(self, key: slice) -> List[O]:
        # pylint: disable=function-redefined; overload
        pass
    def __getitem__(self, key: Union[int, slice, str]) -> Union[O, List[O]]:
        # pylint: disable=function-redefined; overload
        if self.check:
            self.check(key)
        if isinstance(key, str):
            if key not in self:
                raise KeyError()
            return self.app.r.oget(key, default=ReferenceError, expect=self.expect)
        if isinstance(key, slice): # type: ignore[misc]
            return self.app.r.omget([id.decode() for id in self.ids[key]],
                                    default=ReferenceError, expect=self.expect)
        return self.app.r.oget(self.ids[key].decode(), default=ReferenceError,
                               expect=self.expect)

    def __iter__(self) -> Iterator[O]:
        # Optimized and used by count()
        return iter(self[:])

    def __contains__(self, item: object) -> bool:
        if isinstance(item, Object):
            return item.id in self
        return isinstance(item, str) and item.encode() in self.ids

    def json(self, restricted: bool = False, include: bool = False, *, rewrite: RewriteFunc = None,
             slc: slice = None) -> Dict[str, object]:
        """See :meth:`JSONifiable.json`.

        *slc* is a slice of items to include, if any.
        """
        count = len(self)
        if slc:
            items = self[slc]
            start = 0 if cast(Optional[int], slc.start) is None else cast(int, slc.start)
            stop = start + len(items)
        return {
            'count': count,
            **(
                {
                    'items': [item.json(restricted=restricted, include=include, rewrite=rewrite)
                              for item in items],
                    'slice': [start, stop]
                } if slc else {})
        }

class Orderable:
    """Mixin for :class:`Collection` whose items can be ordered.

    The underlying Redis collection must be a Redis list.
    """

    ids = None # type: RedisSequence
    app = None # type: Application

    def move(self, item: Object, to: Optional[Object]) -> None:
        """See :http:post:`/api/(collection-path)/move`."""
        if to:
            if to.id not in self: # type: ignore
                raise ValueError('to_not_found')
            if to == item:
                # No op
                return
        if not cast(int, self.app.r.lrem(self.ids.key, 1, item.id)):
            raise ValueError('item_not_found')
        if to:
            self.app.r.linsert(self.ids.key, 'after', to.id, item.id)
        else:
            self.app.r.lpush(self.ids.key, item.id)

class User(Object, Editable):
    """See :ref:`User`."""

    def __init__(self, *, app: Application, **data: Dict[str, object]) -> None:
        super().__init__(id=cast(str, data['id']), app=app)
        Editable.__init__(self, authors=cast(List[str], data['authors']))
        self.name = cast(str, data['name'])
        self.email = cast(Optional[str], data['email'])
        self.create_time = datetime.fromisoformat(cast(str, data['create_time']))
        self.authenticate_time = datetime.fromisoformat(cast(str, data['authenticate_time']))
        self.devices = Collection(
            RedisSortedSet(f'{self.id}.devices', app.r.r), check=self._check_user,
            expect=expect_type(Device), app=app)

    def store_email(self, email: str) -> None:
        """Update the user's *email* address.

        If *email* is already associated with another user, a :exc:`ValueError`
        (``email_duplicate``) is raised.
        """
        check_email(email)
        id = self.app.r.r.hget('user_email_map', email.encode())
        if id and id.decode() != self.id:
            raise ValueError('email_duplicate')

        if self.email:
            self.app.r.hdel('user_email_map', self.email)
        self.email = email
        self.app.r.oset(self.id, self)
        self.app.r.hset('user_email_map', self.email, self.id)

    def set_email(self, email: str) -> 'AuthRequest':
        """See :http:post:`/api/users/(id)/set-email`."""
        if self.app.user != self:
            raise error.PermissionError()
        check_email(email)

        code = randstr(length=5, charset=string.ascii_uppercase)
        auth_request = AuthRequest(id='AuthRequest:' + randstr(), app=self.app, email=email,
                                   code=code)
        self.app.r.oset(auth_request.id, auth_request)
        self.app.r.expire(auth_request.id, 10 * 60)
        if self.app.render_email_auth_message:
            self._send_email(email, self.app.render_email_auth_message(email, auth_request, code))
        return auth_request

    def finish_set_email(self, auth_request: 'AuthRequest', auth: str) -> None:
        """See :http:post:`/api/users/(id)/finish-set-email`."""
        # pylint: disable=protected-access; auth_request is a friend
        if self.app.user != self:
            raise error.PermissionError()
        auth_request.verify(auth)
        self.app.r.delete(auth_request.id)
        self.store_email(auth_request._email)

    def remove_email(self):
        """See :http:post:`/api/users/(id)/remove-email`."""
        if self.app.user != self:
            raise error.PermissionError()
        if not self.email:
            raise ValueError('user_no_email')

        self.app.r.hdel('user_email_map', self.email)
        self.email = None
        self.app.r.oset(self.id, self)

    def send_email(self, msg):
        """Send an email message to the user.

        *msg* is the message string of the following form: It starts with a line containing the
        subject prefixed with ``Subject:_``, followed by a blank line, followed by the body.

        If the user's ::attr:`email` is not set, a :exc:`ValueError` (``user_no_email``) is raised.
        If communication with the SMTP server fails, an :class:`EmailError` is raised.
        """
        if not self.email:
            raise ValueError('user_no_email')
        self._send_email(self.email, msg)

    def notify(self, event: Event) -> None:
        """Notify the user about *event*.

        The notification is delivered to all enabled devices. If a device's *push_subscription* has
        expired, notifications are disabled for that device.
        """
        devices = self.app.r.omget([id.decode() for id in self.devices.ids], default=AssertionError,
                                   expect=expect_type(Device))
        for device in devices:
            if device.notification_status == 'on':
                create_task(self._notify(device, event))

    def do_edit(self, **attrs):
        if self.app.user != self:
            raise error.PermissionError()

        e = InputError()
        if 'name' in attrs and not str_or_none(attrs['name']):
            e.errors['name'] = 'empty'
        e.trigger()

        if 'name' in attrs:
            self.name = attrs['name']

    def json(self, restricted: bool = False, include: bool = False, *,
             rewrite: RewriteFunc = None) -> dict[str, object]:
        """See :meth:`Object.json`."""
        return {
            **super().json(restricted=restricted, include=include, rewrite=rewrite),
            **Editable.json(self, restricted=restricted, include=include, rewrite=rewrite),
            'name': self.name,
            **(
                {} if restricted and context.user.get() != self else {
                    'email': self.email,
                    'create_time': self.create_time.isoformat(),
                    'authenticate_time': self.authenticate_time.isoformat()
                })
        }

    def _send_email(self, to: str, msg: str) -> None:
        match = re.fullmatch(r'Subject: ([^\n]+)\n\n(.+)', msg, re.DOTALL)
        if not match:
            raise ValueError('msg_invalid')

        msg = EmailMessage()
        msg['To'] = to
        msg['From'] = self.app.email
        msg['Subject'] = match.group(1)
        msg.set_content(match.group(2))

        components = urlparse(self.app.smtp_url)
        host = components.hostname or 'localhost'
        port = components.port or 25
        try:
            with SMTP(host=host, port=port) as smtp:
                smtp.send_message(msg)
        except OSError as e:
            raise EmailError() from e

    async def _notify(self, device: Device, event: Event) -> None:
        try:
            assert device.push_subscription
            await self.app.send_device_notification(device.push_subscription, event)
        except (ValueError, CommunicationError) as e:
            if isinstance(e, ValueError):
                if str(e) != 'push_subscription_invalid':
                    raise e
                device.abort_notifications()
            getLogger(__name__).error('Failed to deliver notification: %s', str(e))

    def _check_user(self, _: Union[int, slice, str]) -> None:
        if context.user.get() != self:
            raise error.PermissionError()

class Users:
    """See :ref:`Users`."""

    def __init__(self, app: Application) -> None:
        self.app = app
        self._ids = RedisList('users', app.r.r)

    def __len__(self) -> int:
        return len(self._ids)

    def __getitem__(self, key: str) -> User:
        if not key.startswith('User:'):
            raise KeyError(key)
        return self.app.r.oget(key, default=KeyError, expect=expect_type(User))

    def __iter__(self) -> Iterator[User]:
        users = self.app.r.omget([id.decode() for id in self._ids], default=AssertionError,
                                 expect=expect_type(User))
        return iter(users)

class Settings(Object, Editable):
    """See :ref:`Settings`.

    .. attribute:: push_vapid_private_key

       VAPID private key used for sending device notifications.
    """

    def __init__(
            self, *, id: str, app: Application, authors: List[str], title: str, icon: Optional[str],
            icon_small: str = None, icon_large: str = None, provider_name: Optional[str],
            provider_url: Optional[str], provider_description: Optional[Dict[str, str]],
            feedback_url: Optional[str], staff: List[str], push_vapid_private_key: str,
            push_vapid_public_key: str) -> None:
        super().__init__(id, app)
        Editable.__init__(self, authors=authors, activity=lambda: app.activity)
        self.title = title
        self.icon = icon
        self.icon_small = icon_small
        self.icon_large = icon_large
        self.provider_name = provider_name
        self.provider_url = provider_url
        self.provider_description = provider_description
        self.feedback_url = feedback_url
        self._staff = staff
        self.push_vapid_private_key = push_vapid_private_key
        self.push_vapid_public_key = push_vapid_public_key

    @property
    def staff(self) -> List[User]:
        # pylint: disable=missing-docstring; already documented
        return self.app.r.omget(self._staff, default=AssertionError, expect=expect_type(User))

    def do_edit(self, **attrs):
        if not self.app.user.id in self._staff:
            raise error.PermissionError()

        e = InputError()
        if 'title' in attrs and not str_or_none(attrs['title']):
            e.errors['title'] = 'empty'
        e.trigger()

        if 'title' in attrs:
            self.title = attrs['title']
        if 'icon' in attrs:
            self.icon = str_or_none(attrs['icon'])
        if 'icon_small' in attrs:
            self.icon_small = str_or_none(attrs['icon_small'])
        if 'icon_large' in attrs:
            self.icon_large = str_or_none(attrs['icon_large'])
        if 'provider_name' in attrs:
            self.provider_name = str_or_none(attrs['provider_name'])
        if 'provider_url' in attrs:
            self.provider_url = str_or_none(attrs['provider_url'])
        if 'provider_description' in attrs:
            self.provider_description = attrs['provider_description']
        if 'feedback_url' in attrs:
            self.feedback_url = str_or_none(attrs['feedback_url'])

    def json(self, restricted=False, include=False, *, rewrite=None):
        return {
            **super().json(restricted=restricted, include=include, rewrite=rewrite),
            **Editable.json(self, restricted=restricted, include=include, rewrite=rewrite),
            'title': self.title,
            'icon': self.icon,
            'icon_small': self.icon_small,
            'icon_large': self.icon_large,
            'provider_name': self.provider_name,
            'provider_url': self.provider_url,
            'provider_description': self.provider_description,
            'feedback_url': self.feedback_url,
            'staff': [u.json(restricted=restricted, rewrite=rewrite) for u in self.staff] if include
                     else self._staff,
            'push_vapid_public_key': self.push_vapid_public_key,
            **({} if restricted else {'push_vapid_private_key': self.push_vapid_private_key})
        }

class Activity(Object, JSONRedisSequence[JSONifiable]):
    """See :ref:`Activity`.

    .. attribute:: post

       Hook of the form ``post(event)`` that is called after :meth:`publish` with the corresponding
       *event*. May be ``None``.
    """

    class Stream(AsyncIterator['Event']):
        """:cls:`collections.abc.AsyncGenerator` of events."""
        # Syntax for Python 3.6+:
        #
        # async def stream(self) -> AsyncGenerator['Event', None]:
        #     """Return a live stream of events."""
        #     queue = Queue() # type: Queue[Event]
        #     self._streams.add(queue)
        #     try:
        #         while True:
        #             yield await cast(Coroutine[object, None, Event], queue.get())
        #     except GeneratorExit as e:
        #         self._streams.remove(queue)
        #         # Work around https://bugs.python.org/issue34730
        #         queue.put_nowait(None) # type: ignore

        def __init__(self, streams: Set['Queue[Optional[Event]]']) -> None:
            self._queue = Queue() # type: Queue[Optional[Event]]
            self._streams = streams
            self._streams.add(self._queue)

        async def asend(self, value: None) -> 'Event':
            # pylint: disable=unused-argument, missing-docstring; part of API
            event = await cast(Coroutine[object, None, Optional[Event]], self._queue.get())
            if event is None:
                raise StopAsyncIteration()
            return event

        async def athrow(self, typ: Type[BaseException], val: BaseException = None,
                         tb: object = None) -> 'Event':
            # pylint: disable=unused-argument, missing-docstring; part of API
            raise val if val else typ()

        async def aclose(self) -> None:
            # pylint: disable=missing-docstring; part of API
            self._streams.remove(self._queue)
            self._queue.put_nowait(None)

        async def __anext__(self) -> 'Event':
            return await self.asend(None)

    def __init__(self, *, app: Application, pre: Callable[[], None] = None, **data: object) -> None:
        id = cast(str, data['id'])
        super().__init__(id=id, app=app)
        JSONRedisSequence.__init__(self, app.r, '{}.items'.format(id), pre)
        self.post = None # type: Optional[Callable[[Event], None]]
        self.host = None # type: Optional[object]
        self._subscriber_ids = cast('list[str]', data['subscriber_ids'])
        self._streams = set() # type: Set[Queue[Optional[Event]]]

    @property
    def subscribers(self) -> List[User]:
        """List of :class:`User`s who subscribed to the activity."""
        return self.app.r.omget(self._subscriber_ids, default=AssertionError,
                                expect=expect_type(User))

    def publish(self, event: 'Event') -> None:
        """Publish an *event* to the feed.

        All :attr:`subscribers`, except the user who triggered the event, are notified.
        """
        # If the event is published to multiple activity feeds, it is stored (and overwritten)
        # multiple times, but that's acceptable for a more convenient API
        self.r.oset(event.id, event)
        self.r.lpush(self.list_key, event.id)
        for subscriber in self.subscribers:
            if subscriber is not event.user:
                subscriber.notify(event)
        for stream in self._streams:
            stream.put_nowait(event)
        if self.post:
            self.post(event)

    def subscribe(self):
        """See :http:patch:`/api/(activity-url)` (``subscribe``)."""
        if not self.app.user:
            raise error.PermissionError()
        if not self.app.user.id in self._subscriber_ids:
            self._subscriber_ids.append(self.app.user.id)
        self.app.r.oset(self.host.id if self.host else self.id, self.host or self)

    def unsubscribe(self):
        """See :http:patch:`/api/(activity-url)` (``unsubscribe``)."""
        if not self.app.user:
            raise error.PermissionError()
        try:
            self._subscriber_ids.remove(self.app.user.id)
        except ValueError:
            pass
        self.app.r.oset(self.host.id if self.host else self.id, self.host or self)

    def stream(self) -> 'Activity.Stream':
        """Return a live stream of events."""
        return Activity.Stream(self._streams)

    def json(self, restricted: bool = False, include: bool = False, *, rewrite: RewriteFunc = None,
             slice: 'slice' = None) -> Dict[str, object]:
        # pylint: disable=arguments-differ; extension
        return {
            **super().json(restricted=restricted, include=include, rewrite=rewrite),
            **({'subscriber_ids': self._subscriber_ids} if not restricted else {}),
            **({'user_subscribed': self.app.user and self.app.user.id in self._subscriber_ids}
               if restricted else {}),
            **(
                {
                    'items': [event.json(restricted=True, include=True, rewrite=rewrite)
                              for event in self[slice]]
                } if slice else {})
        }

class Event(Object):
    """See :ref:`Event`."""

    @staticmethod
    def create(type: str, object: Optional[Object], detail: Dict[str, object] = {},
               app: Application = None) -> 'Event':
        """Create an event."""
        assert app
        if not app.user:
            raise error.PermissionError()
        if not str_or_none(type):
            raise ValueError('type_empty')
        if any(k.endswith('_id') for k in detail):
            raise ValueError('detail_invalid_key')

        transformed = {}
        for key, value in detail.items():
            if isinstance(value, Object):
                key = key + '_id'
                value = value.id
            transformed[key] = value
        return Event(
            id='Event:' + randstr(), type=type, object=object.id if object else None,
            user=app.user.id, time=datetime.utcnow().isoformat() + 'Z', detail=transformed, app=app)

    def __init__(self, id: str, type: str, object: Optional[str], user: str, time: str,
                 detail: Dict[str, object], app: Application) -> None:
        super().__init__(id, app)
        self.type = type
        self.time = datetime.fromisoformat(time.replace('Z', '+00:00'))
        self._object_id = object
        self._user_id = user
        self._detail = detail

    @property
    def object(self) -> Optional[Union[Object, Gone]]:
        # pylint: disable=missing-docstring; already documented
        return (
            self.app.r.oget(self._object_id, expect=expect_type(Object)) or Gone()
            if self._object_id else None)

    @property
    def user(self) -> User:
        # pylint: disable=missing-docstring; already documented
        return self.app.users[self._user_id]

    @property
    def detail(self) -> Dict[str, builtins.object]:
        # pylint: disable=missing-docstring; already documented
        detail = {}
        for key, value in self._detail.items():
            if key.endswith('_id'):
                assert isinstance(value, str)
                key = key[:-3]
                value = self.app.r.oget(value) or Gone()
            detail[key] = value
        return detail

    def json(self, restricted: bool = False, include: bool = False, *,
             rewrite: RewriteFunc = None) -> Dict[str, builtins.object]:
        obj = self._object_id # type: Optional[Union[str, Dict[str, object]]]
        detail = self._detail
        if include:
            obj = (self.object.json(restricted=restricted, include=include, rewrite=rewrite)
                   if self.object else None)
            detail = {
                k: v.json(restricted=restricted, include=include, rewrite=rewrite)
                   if isinstance(v, (Object, Gone)) else v for k, v in self.detail.items()
            }
        return {
            **super().json(restricted=restricted, include=include, rewrite=rewrite),
            'type': self.type,
            'object': obj,
            'user': self.user.json(restricted=restricted, rewrite=rewrite) if include
                    else self._user_id,
            'time': self.time.isoformat(),
            'detail': detail
        }

    def __str__(self) -> str:
        return '<{} {} on {} by {}>'.format(type(self).__name__, self.type, self._object_id,
                                            self._user_id)
    __repr__ = __str__

class AuthRequest(Object):
    """See :ref:`AuthRequest`."""

    def __init__(self, id: str, app: Application, email: str, code: str) -> None:
        super().__init__(id, app)
        self._email = email
        self._code = code

    def verify(self, code: str) -> None:
        """Verify the secret *code*."""
        self.app.rate_limiter.count(RateLimit(f'{self.id}.verify', 10, timedelta(minutes=10)),
                                    context.client.get())
        if code != self._code:
            raise ValueError('Invalid code')

    def json(self, restricted: bool = False, include: bool = False, *,
             rewrite: RewriteFunc = None) -> Dict[str, object]:
        return {
            **super().json(restricted=restricted, include=include, rewrite=rewrite),
            **({} if restricted else {'email': self._email, 'code': self._code})
        }

class Location:
    """See :ref:`Location`."""

    def __init__(self, name: str, coords: Tuple[float, float] = None) -> None:
        if str_or_none(name) is None:
            raise ValueError('empty_name')
        if coords and not (-90 <= coords[0] <= 90 and -180 <= coords[1] <= 180):
            raise ValueError('out_of_range_coords')
        self.name = name
        self.coords = coords

    @staticmethod
    def parse(data: Dict[str, object]) -> 'Location':
        """Parse the given location JSON *data* into a :class:`Location`."""
        coords_list = Expect.opt(Expect.list(Expect.float))(data.get('coords'))
        if coords_list is None:
            coords = None
        else:
            if len(coords_list) != 2:
                raise TypeError()
            coords = (float(coords_list[0]), float(coords_list[1]))
        return Location(Expect.str(data.get('name')), coords)

    def json(self) -> Dict[str, object]:
        """Return a JSON representation of the location."""
        return {'name': self.name, 'coords': list(self.coords) if self.coords else None}

class InputError(ValueError):
    """See :ref:`InputError`.

    To raise an :exc:`InputError`, apply the following pattern::

       def meow(volume):
           e = InputError()
           if not 0 < volume <= 1:
               e.errors['volume'] = 'out_of_range'
           e.trigger()
           # ...
    """

    def __init__(self, errors: Dict[str, str] = {}) -> None:
        super().__init__('input_invalid')
        self.errors = dict(errors)

    def trigger(self):
        """Trigger the error, i.e. raise it if any *errors* are present.

        If *errors* is empty, do nothing.
        """
        if self.errors:
            raise self

class EmailError(Exception):
    """Raised if communication with the SMTP server fails."""


import json
import logging
import sys

from passivetotal.common.utilities import is_ip

log = logging.getLogger('passivetotal')
log.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
log.addHandler(ch)

misperrors = {'error': 'Error'}
mispattributes = {
    'input': ['hostname', 'domain', 'ip-src', 'ip-dst',
              'x509-fingerprint-sha1', 'email-src', 'email-dst',
              'target-email', 'whois-registrant-email',
              'whois-registrant-phone', 'text', 'whois-registrant-name',
              'whois-registrar', 'whois-creation-date'],
    'output': ['hostname', 'domain', 'ip-src', 'ip-dst',
               'x509-fingerprint-sha1', 'email-src', 'email-dst',
               'target-email', 'whois-registrant-email',
               'whois-registrant-phone', 'text', 'whois-registrant-name',
               'whois-registrar', 'whois-creation-date', 'md5', 'sha1',
               'sha256', 'link']
}
moduleinfo = {
    'version': '1.0',
    'author': 'Brandon Dixon',
    'description': 'The PassiveTotal MISP expansion module brings the datasets derived from Internet scanning directly into your MISP instance. This module supports passive DNS, historic SSL, WHOIS, and host attributes. In order to use the module, you must have a valid PassiveTotal account username and API key. Registration is free and can be done by visiting https://www.passivetotal.org/register',
    'module-type': ['expansion', 'hover']
}
moduleconfig = ['username', 'api_key']
query_playbook = [
    {'inputs': ['ip-src', 'ip-dst', 'hostname', 'domain'],
     'services': ['whois', 'ssl', 'dns', 'enrichment'],
     'name': 'generic'},
    {'inputs': ['whois-registrant-email', 'whois-registrant-phone',
                'whois-registrant-name', 'email-src', 'email-dst',
                'target-email'],
     'services': ['whois'],
     'name': 'reverse-whois'},
    {'inputs': ['x509-fingerprint-sha1'],
     'services': ['ssl'],
     'name': 'ssl-history'},
]

def query_finder(request):
    """Find the query value in the client request."""
    for item in mispattributes['input']:
        if not request.get(item, None):
            continue

        playbook = None
        for x in query_playbook:
            if item not in x['inputs']:
                continue
            playbook = x
            break

        return {'type': item, 'value': request.get(item), 'playbook': playbook}

def build_profile(request):
    """Check the incoming request for a valid configuration."""
    output = {'success': False}
    config = request.get('config', None)
    if not config:
        misperrors['error'] = "Configuration is missing from the request."
        return output

    for item in moduleconfig:
        if config.get(item, None):
            continue
        misperrors['error'] = "PassiveTotal authentication is missing."
        return output

    profile = {'success': True, 'config': config}
    profile.update(query_finder(request))

    return profile

def _generate_request_instance(conf, request_type):
    """Automatically generate a request instance to use.

    In the end, this saves us from having to load each request class in a
    explicit way. Loading via a string is helpful to reduce the code per
    call.

    :param request_type: Type of client to load
    :return: Loaded PassiveTotal client
    """
    pt_username = conf.get('username')
    pt_api_key = conf.get('api_key')

    class_lookup = {'dns': 'DnsRequest', 'whois': 'WhoisRequest',
                    'ssl': 'SslRequest', 'enrichment': 'EnrichmentRequest',
                    'attributes': 'AttributeRequest'}
    class_name = class_lookup[request_type]
    mod = __import__('passivetotal.libs.%s' % request_type,
                     fromlist=[class_name])
    loaded = getattr(mod, class_name)
    headers = {'PT-INTEGRATION': 'MISP'}
    authenticated = loaded(pt_username, pt_api_key, headers=headers)
    return authenticated

def _has_error(results):
    """Check to see if there's an error in place and log it."""
    if 'error' in results:
        msg = "%s - %s" % (results['error']['message'],
                           results['error']['developer_message'])
        misperrors['error'] = msg
        return True

    return False

def process_ssl_details(instance, query):
    """Process details for a specific certificate."""
    log.debug("SSL Details: starting")
    values = list()
    details = instance.get_ssl_certificate_details(query=query)
    err = _has_error(details)
    if err:
        raise Exception("We hit an error, time to bail!")
    if details.get('message') and details['message'].startswith('quota_exceeded'):
        raise Exception("API quota exceeded.")
    values = {value for value in details.values() if value}
    txt = [{'types': ['ssl-cert-attributes'], 'values': list(values)}]
    log.debug("SSL Details: ending")

    return txt

def process_ssl_history(instance, query):
    """Process the history for an SSL certificate."""
    log.debug("SSL History: starting")

    type_map = {
        'ip': ['ip-src', 'ip-dst'],
        'domain': ['domain', 'hostname'],
        'sha1': ['x509-fingerprint-sha1']
    }

    hits = {'ip': list(), 'sha1': list(), 'domain': list()}
    history = instance.get_ssl_certificate_history(query=query)
    err = _has_error(history)
    if err:
        raise Exception("We hit an error, time to bail!")
    if history.get('message') and history['message'].startswith('quota_exceeded'):
        raise Exception("API quota exceeded.")
    for item in history.get('results', []):
        hits['ip'] += item.get('ipAddresses', [])
        hits['sha1'].append(item['sha1'])
        hits['domain'] += item.get('domains', [])

    tmp = list()
    for key, value in hits.items():
        txt = {'types': type_map[key], 'values': list(set(value))}
        tmp.append(txt)

    log.debug("SSL Details: ending")

    return tmp

def process_whois_details(instance, query):
    """Process the detail from the WHOIS record."""
    log.debug("WHOIS Details: starting")
    tmp = list()
    details = instance.get_whois_details(query=query, compact_record=True)
    err = _has_error(details)
    if err:
        raise Exception("We hit an error, time to bail!")
    if details.get('message') and details['message'].startswith('quota_exceeded'):
        raise Exception("API quota exceeded.")
    if details.get('contactEmail', None):
        tmp.append({'types': ['whois-registrant-email'], 'values': [details.get('contactEmail')]})
    phones = details['compact']['telephone']['raw']
    tmp.append({'types': ['whois-registrant-phone'], 'values': phones})
    names = details['compact']['name']['raw']
    tmp.append({'types': ['whois-registrant-name'], 'values': names})
    if details.get('registrar', None):
        tmp.append({'types': ['whois-registrar'], 'values': [details.get('registrar')]})
    if details.get('registered', None):
        tmp.append({'types': ['whois-creation-date'], 'values': [details.get('registered')]})
    log.debug("WHOIS Details: ending")

    return tmp

def process_whois_search(instance, query, qtype):
    """Process a WHOIS search for a specific field value."""
    log.debug("WHOIS Search: starting")
    if qtype in ['whois-registrant-email', 'email-src', 'email-dst', 'target-email']:
        field_type = 'email'
    if qtype in ['whois-registrant-phone']:
        field_type = 'phone'
    if qtype in ['whois-registrant-name']:
        field_type = 'name'

    domains = list()
    search = instance.search_whois_by_field(field=field_type, query=query)
    err = _has_error(search)
    if err:
        raise Exception("We hit an error, time to bail!")
    if search.get('message') and search['message'].startswith('quota_exceeded'):
        raise Exception("API quota exceeded.")
    for item in search.get('results', []):
        domain = item.get('domain', None)
        if not domain:
            continue
        domains.append(domain)

    tmp = [{'types': ['hostname', 'domain'], 'values': list(set(domains))}]
    log.debug("WHOIS Search: ending")

    return tmp

def process_passive_dns(instance, query):
    """Process passive DNS data."""
    log.debug("Passive DNS: starting")
    tmp = list()
    pdns = instance.get_unique_resolutions(query=query)
    err = _has_error(pdns)
    if err:
        raise Exception("We hit an error, time to bail!")
    if pdns.get('message') and pdns['message'].startswith('quota_exceeded'):
        raise Exception("API quota exceeded.")
    if is_ip(query):
        tmp = [{'types': ['domain', 'hostname'], 'values': pdns.get('results', [])}]
    else:
        tmp = [{'types': ['ip-src', 'ip-dst'], 'values': pdns.get('results', [])}]
    log.debug("Passive DNS: ending")

    return tmp

def process_osint(instance, query):
    """Process OSINT links."""
    log.debug("OSINT: starting")
    urls = list()
    osint = instance.get_osint(query=query)
    err = _has_error(osint)
    if err:
        raise Exception("We hit an error, time to bail!")
    if osint.get('message') and osint['message'].startswith('quota_exceeded'):
        raise Exception("API quota exceeded.")
    for item in osint.get('results', []):
        urls.append(item['sourceUrl'])

    tmp = [{'types': ['link'], 'values': urls}]
    log.debug("OSINT: ending")

    return tmp

def process_malware(instance, query):
    """Process malware samples."""
    log.debug("Malware: starting")
    content = {'hashes': list(), 'urls': list()}
    malware = instance.get_malware(query=query)
    err = _has_error(malware)
    if err:
        raise Exception("We hit an error, time to bail!")
    if malware.get('message') and malware['message'].startswith('quota_exceeded'):
        raise Exception("API quota exceeded.")
    for item in malware.get('results', []):
        content['hashes'].append(item['sample'])
        content['urls'].append(item['sourceUrl'])

    tmp = [{'types': ['link'], 'values': content['urls']}]
    hashes = {'md5': list(), 'sha1': list(), 'sha256': list()}
    for h in content['hashes']:
        if len(h) == 32:
            hashes['md5'].append(h)
        elif len(h) == 41:
            hashes['sha1'].append(h)
        elif len(h) == 64:
            hashes['sha256'].append(h)
    tmp += [{'types': ['md5'], 'values': hashes['md5']}]
    tmp += [{'types': ['sha1'], 'values': hashes['sha1']}]
    tmp += [{'types': ['sha256'], 'values': hashes['sha256']}]
    log.debug("Malware: ending")

    return tmp

def handler(q=False):
    if not q:
        return q

    request = json.loads(q)
    profile = build_profile(request)
    if not profile['success']:
        log.error(misperrors['error'])
        return misperrors

    output = {'results': list()}

    instances = dict()
    for service in profile['playbook']['services']:
        instances[service] = _generate_request_instance(
            profile['config'], service)

    play_type = profile['playbook']['name']
    query = profile['value']
    qtype = profile['type']
    try:
        if play_type == 'generic':
            results = process_passive_dns(instances['dns'], query)
            output['results'] += results
            results = process_whois_details(instances['whois'], query)
            output['results'] += results
            results = process_ssl_history(instances['ssl'], query)
            output['results'] += results
            results = process_osint(instances['enrichment'], query)
            output['results'] += results
            results = process_malware(instances['enrichment'], query)
            output['results'] += results
        elif play_type == 'reverse-whois':
            results = process_whois_search(instances['whois'], query, qtype)
            output['results'] += results
        elif play_type == 'ssl-history':
            results = process_ssl_details(instances['ssl'], query)
            output['results'] += results
            results = process_ssl_history(instances['ssl'], query)
            output['results'] += results
        else:
            log.error("Unsupported query pattern issued.")
    except Exception as e:
        misperrors['error'] = e.__str__()
        return misperrors

    return output

def introspection():
    return mispattributes

def version():
    moduleinfo['config'] = moduleconfig
    return moduleinfo


import flet as ft
import random

def main(page: ft.Page):
    page.add(
    t = ft.Text(value="ELIGE PIEDRA PAPEL O TIJERA", color="pink")
    )
    opciones = ["piedra", "papel", "tijera"]
    opcion_compu = random.choice(opciones)
    usuario = input("piedra, papel o tijera: ").lower()
    if usuario not in opciones:
        print("Elige una opción válida")
        return
    print(f"Tu: {usuario}")
    print(f"Compu: {opcion_compu}")
    if usuario == opcion_compu:
        print("Empate")
    elif (usuario == "piedra" and opcion_compu == "tijera") or (usuario == "papel" and opcion_compu == "piedra") or (usuario == "tijera" and opcion_compu == "papel"):
        print("Eres un ganador")
    else:
        print("Eres perdedor!")

ft.app(target=main)


import os

this_dir = os.path.dirname(os.path.abspath(__file__))

import numpy as np
import torch as T
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from gymnasium.utils.save_video import save_video
from utils.plotting import plot_learning_curve

class DQN(nn.Module):
    def __init__(self, env, lr, fc1_dims, fc2_dims, fc3_dims):
        super(DQN,self).__init__()
        self.input_dims = [env.observation_space.shape[0]]
        self.fc1_dims = fc1_dims
        self.fc2_dims = fc2_dims
        self.fc3_dims = fc3_dims
        self.n_actions = env.action_space.n
        self.fc1 = nn.Linear(*self.input_dims,self.fc1_dims)
        self.fc2 = nn.Linear(self.fc1_dims,self.fc2_dims)
        self.fc3 = nn.Linear(self.fc2_dims,self.fc3_dims)
        self.fc4 = nn.Linear(self.fc3_dims,self.n_actions)
        self.optimizer = optim.Adam(self.parameters(), lr=lr)
        self.loss = nn.HuberLoss(delta=0.8)
        self.device = T.device('cuda:0' if T.cuda.is_available() else 'cpu')
        self.to(self.device)

    def forward(self,state):
        x = F.tanh(self.fc1(state))
        x = F.tanh(self.fc2(x))
        x = F.tanh(self.fc3(x))
        actions = self.fc4(x)
        return actions

class DQNAgent():
    def __init__(self, env, n_episodes=100, gamma=0.99, epsilon=1.0, lr=1e-3, batch_size=100, buffer_size=100000, max_steps=700, eps_end=0.01, eps_dec=5e-4):
        # Setup agent
        self.env = env
        self.n_episodes = n_episodes
        self.gamma = gamma
        self.epsilon = epsilon
        self.lr = lr
        self.eps_min = eps_end
        self.eps_dec = eps_dec
        self.action_space = list(range(env.action_space.n))
        self.buffer_size = buffer_size
        self.batch_size = batch_size
        self.max_steps = max_steps
        self.mem_cnter = 0
        self.Q_eval = DQN(env, lr, fc1_dims=200, fc2_dims=200, fc3_dims=200)

        #build memory holders
        self.state_memory = np.zeros((self.buffer_size, *[env.observation_space.shape[0]]), dtype=np.float32) # build memory holder
        self.new_state_memory = np.zeros((self.buffer_size, *[env.observation_space.shape[0]]), dtype=np.float32) # build memory holder
        self.action_memory = np.zeros(self.buffer_size, dtype=np.int32)
        self.reward_memory =np.zeros(self.buffer_size, dtype=np.float32)
        self.terminal_memory = np.zeros(self.buffer_size, dtype = np.bool_)

        # Stuff for data, plots, and videos
        self.step_starting_index = 0
        self.dqn_data_path = os.path.abspath(os.path.join(this_dir, '..', '..', 'data', 'dqn'))
        self.video_path = os.path.abspath(os.path.join(self.dqn_data_path, 'videos'))
        self.plots_path = os.path.abspath(os.path.join(self.dqn_data_path, 'plots'))

    def reset_memory(self):
        self.mem_cnter = 0
        self.state_memory = np.zeros((self.buffer_size, *[self.env.observation_space.shape[0]]), dtype=np.float32)
        self.new_state_memory = np.zeros((self.buffer_size, *[self.env.observation_space.shape[0]]), dtype=np.float32)
        self.action_memory = np.zeros(self.buffer_size, dtype=np.int32)
        self.reward_memory = np.zeros(self.buffer_size, dtype=np.float32)
        self.terminal_memory = np.zeros(self.buffer_size, dtype=np.bool_)

    # Store experience into buffer
    def store_transition(self,state,action,reward,state_,terminated):
        index = self.mem_cnter % self.buffer_size # can overwrite oldest memories
        self.state_memory[index] = state
        self.new_state_memory[index] = state_
        self.reward_memory[index]=  reward
        self.terminal_memory[index] = terminated
        self.action_memory[index] = action
        self.mem_cnter += 1

    # Epsilon-Greedy Policy
    def select_action(self,observation):
        if np.random.random() > self.epsilon:
            state = T.Tensor(np.array(observation)).to(self.Q_eval.device)
            actions = self.Q_eval.forward(state)
            action = T.argmax(actions).item() # convert from tensor
        else:
            action = np.random.choice(self.action_space)
        return action

    def learn(self):

        # Just explore until the batch is full
        if self.mem_cnter < self.batch_size:
            return 0.0

        self.Q_eval.optimizer.zero_grad()

        max_mem = min(self.mem_cnter, self.buffer_size) # take the less full one
        batch = np.random.choice(max_mem, self.batch_size, replace=False) # get rid of duplicate memories for learning

        batch_index = np.arange(self.batch_size, dtype=np.int32)

        state_batch = T.tensor(self.state_memory[batch]).to(self.Q_eval.device) # turn a subset of memory into a pytorch tensor
        new_state_batch = T.tensor(self.new_state_memory[batch]).to(self.Q_eval.device)
        reward_batch = T.tensor(self.reward_memory[batch]).to(self.Q_eval.device)
        terminal_batch = T.tensor(self.terminal_memory[batch]).to(self.Q_eval.device)
        action_batch = self.action_memory[batch]

        # drive agent to max action
        q_eval = self.Q_eval.forward(state_batch)[batch_index,action_batch] # grab actions you took
        q_next = self.Q_eval.forward(new_state_batch)# get next actions
        q_next[terminal_batch] = 0.0
        q_target = reward_batch+self.gamma*T.max(q_next,dim=1)[0] # grab value not index

        loss = self.Q_eval.loss(q_target,q_eval).to(self.Q_eval.device)
        loss.backward() # back propagate

        self.Q_eval.optimizer.step()

        # decay epsilon
        self.epsilon = self.epsilon - self.eps_dec if self.epsilon > self.eps_min \
            else self.eps_min

        return loss.item()

    def train(self, msim=None):
        # Initialize metrics
        scores, losses, stds, avg_scores, avg_losses, eps_history = [], [], [], [], [], []
        best_score = float('-inf')

        # Loop over the range of the number of episodes
        for ep_index in range(self.n_episodes):
            # Initialize the experience for this episode
            score = 0
            terminated = False
            truncated = False
            obs = self.env.reset()
            s = obs[0]
            env_step = 0

            while not terminated and not truncated and env_step < self.max_steps:
                action = self.select_action(s)

                # Gather experience
                sp, r, terminated, truncated, _ = self.env.step(action)

                # Increment score
                score += r

                # Store experience into buffer
                self.store_transition(s, action, r, sp, terminated)
                loss = self.learn()

                # Setup for next iteration
                s = sp
                env_step += 1

                # Save environment rendering
                if (terminated or truncated or env_step==self.max_steps) and msim is None:
                    env_render = self.env.render()
                    save_video(
                        env_render,
                        video_folder=self.video_path,
                        fps=self.env.metadata["render_fps"],
                        name_prefix="dqn_video",
                        step_starting_index=self.step_starting_index,
                        episode_index=ep_index
                    )
                    self.step_starting_index += env_step
                    env_step = 0
                    if ep_index==self.n_episodes-1:
                        save_video(
                            env_render,
                            video_folder=self.video_path,
                            fps=self.env.metadata["render_fps"],
                            name_prefix="final_dqn_video",
                        )

            if score > best_score and msim is None:
                best_score = score
                if best_score > 100.0:
                    save_video(
                        env_render,
                        video_folder=self.video_path,
                        name_prefix=f'best_dqn_score{best_score:.2f}',
                        fps=self.env.metadata["render_fps"],
                    )

            # Store the score for this episode
            scores.append(score)
            losses.append(loss)
            eps_history.append(self.epsilon)

            # Compute average score and std over the last 25 episodes
            avg_loss = np.mean(losses[-25:])
            avg_score = np.mean(scores[-25:])
            avg_scores.append(avg_score)
            avg_losses.append(avg_loss)
            std = np.std(scores[-25:])
            stds.append(std)

            print('episode', ep_index, 'score %.2f' % score,'avg score %.2f' % avg_score)

        if msim is not None:
            plot_learning_curve(list(range(1, self.n_episodes+1)), losses, avg_losses, self.plots_path + f'/loss_learning_curve_mc_{msim+1}.png', 'loss')
            plot_learning_curve(list(range(1, self.n_episodes+1)), scores, avg_scores, self.plots_path + f'/score_learning_curve_mc_{msim+1}.png', 'score')
        else:
            plot_learning_curve(list(range(1, self.n_episodes+1)), losses, avg_losses, self.plots_path + f'/loss_learning_curve.png', 'loss')
            plot_learning_curve(list(range(1, self.n_episodes+1)), scores, avg_scores, self.plots_path + f'/score_learning_curve.png', 'score')

        return scores, avg_score, std


from functools import lru_cache

from yaml import safe_load

from ansible_collections.arista.avd.plugins.plugin_utils.schema.default_schemas import DEFAULT_SCHEMAS

@lru_cache
def create_store():
    store = {}
    for id, schema_file in DEFAULT_SCHEMAS.items():
        with open(schema_file, "r", encoding="UTF-8") as file:
            schema_file_data = safe_load(file.read())
            store[id] = schema_file_data

    return store


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os
import argparse
import pickle

import hjson as json
import torch

import loader
from calc_F_score import calc_F_score  # ~/work/misc
from model.AutoNER_with_RL import AutoNER_with_RL

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='This script is BiLSTM-CRF \
        trainable imcomplete annotation')
    parser.add_argument('--model', type=str, default=None,
                        help='load a model information, please input file stem \
        without ".info"/".weight"')
    parser.add_argument('--test_file', type=str, default=None,
                        help='test corpus')
    parser.add_argument('--minibatch', type=int, default=32)
    parser.add_argument('--token_splitter', type=str, default=' ',
                        help="that's/O a/O apple-pen/PPAP -> \
        [that's/O, a/O, apple-pen/PP|AP]")
    parser.add_argument('--tag_splitter', type=str, default='/',
                        help=" -> [O, O, PP|AP]")
    args = parser.parse_args()

    test_data = open(args.test_file, 'r').read().split('\n')
    test_data = [_.split(args.token_splitter) for _ in test_data if _ != '']

    with open(args.model + '.info.hjson', mode='rb') as f:
        model_info = pickle.load(f)
    autoner = AutoNER_with_RL(model_info['prams'],
                              model_info['w2i'],
                              model_info['i2w'],
                              model_info['ch2i'],
                              model_info['i2ch'],
                              model_info['t2i'],
                              model_info['i2t'])
    autoner.load_state_dict(torch.load(args.model + '.weight',
                                       map_location=lambda storage, loc: storage))
    if 'CUDA_VISIBLE_DEVICES' in os.environ:
        gpu_id = os.environ['CUDA_VISIBLE_DEVICES']
        if int(gpu_id) >= 0:
            cuda_device = torch.device('cuda')
            autoner.to(cuda_device)
            autoner.to_gpu(cuda_device)

    dev_sents, dev_sent_labels = \
        loader.read_corpus_column(args.test_file, {}, {}, {}, {}, {}, {}, {}, {},
                                  '\n\n', '\n', ' ', None, 0, 0, None,
                                  vocab_expand=False, return_BIESO=True)
    dev_corpus = [dev_sents, dev_sent_labels]
    for i, (sent, sent_label) in enumerate(zip(dev_sents, dev_sent_labels)):
        if len(sent) == 0:
            dev_sents.remove(sent)
            dev_sent_labels.remove(sent_label)
    assert(len(dev_corpus[0]) == len(dev_corpus[1]))

    minibatch = args.minibatch
    autoner.eval()
    predict_sents = []
    dev_sents = dev_corpus[0]
    for i in range(0, len(dev_sents), minibatch):
        input_sents = []
        for r in range(i, min(i+minibatch, len(dev_corpus[0]))):
            input_sents.append(dev_sents[r])
        predicts = autoner.predict(input_sents)
        predict_sents.extend(predicts)
    prec, recall, f_score = calc_F_score(dev_corpus[1],
                                         predict_sents,
                                         ' ',
                                         '</>',
                                         'O',
                                         '-',
                                         0)
    print(prec, recall, f_score)
    # for words in test_data:
    #     predict_tags = model.predict([words])
    #     newline = ''
    #     for t, word in enumerate(words):
    #         newline += word + args.tag_splitter + predict_tags[t] + \
    #             args.token_splitter
    #     newline = newline[:-len(args.token_splitter)]
    #     print(newline)


from django.contrib import admin
from .models import *

# Register your models here.

our_models = [
    Biography,
    Project,
    ContactMe,
    SocialsProfile,
    Profile,
    ProjectNotes,
    ResumeProjects,
    ResumeProjectDetails,
    Resume,
    ResumeAwardsAndAchievements,
    CurrProj,
    Feedback,
    BiographySection,
    BiographySectionImage,
]

admin.site.register(model for model in our_models)


import praw
import requests
import json
from datetime import datetime
import time
import argparse
from os.path import join
from os import getcwd
from tqdm import tqdm

class Downloader:
    def __init__(self, client_id = None, client_secret = None, user_agent = None):
        if client_id is not None:
            self.reddit = praw.Reddit(client_id=client_id,
                                client_secret=client_secret,
                                user_agent=user_agent)

    def scrape_posts(self, subreddit, start_time = None, end_time = None, interval = 3600*6):
        '''With default args, downloads all posts from a subreddit
        For optimal speed, interval should be big enough that there are more post
        than the pushshift_max (currently 100) during that period.
        '''
        wait_time = 0.5
        retries = 10
        pushshift_max = 100
        if start_time is None:
            if not hasattr(self, "reddit"):
                raise Exception("Since no API keys were provided, start_time and end_time must be specified")
            start_time = int(self.reddit.subreddit(subreddit).created_utc)
        if end_time is None:
            end_time = int(datetime.now().timestamp())

        output_file = f'fetched_r_{subreddit}_{start_time}_{end_time}.json'
        with open(output_file, 'w') as f:
            print(f"Fetching data and saving it in {join(getcwd(),output_file)}...")
            f.write("[")
            tqdmbar = tqdm(total = end_time - start_time)
            while start_time < end_time:
                end_interval = min(start_time + interval, end_time)
                url = f"https://api.pushshift.io/reddit/search/submission/?after={start_time}&before={end_interval}&subreddit={subreddit}&limit=1000&score=%3E0"
                r = requests.get(url)
                for i in range(retries):
                    if r.status_code == 200:
                        break
                    else:
                        print("ERROR", str(r.status_code))
                        time.sleep(1*(2**i))
                        r = requests.get(url)
                posts = json.loads(r.text)["data"]
                if len(posts) < pushshift_max:
                    tqdmbar.update(interval)
                    start_time += interval
                else:
                    tqdmbar.update(int(posts[-1]["created_utc"]) - start_time)
                    start_time = int(posts[-1]["created_utc"])
                for post in posts:
                    json.dump(post, f)
                    f.write(",\n")
                time.sleep(wait_time)
            tqdmbar.close()
            # correcting end of file:
            f.seek(f.tell()-2)
            f.truncate()
            f.write("]")

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--subreddit', type=str, required = True)
    parser.add_argument('--client_id', type=str, default = None)
    parser.add_argument('--client_secret', type=str, default = None)
    parser.add_argument('--user_agent', type=str, default = None)
    parser.add_argument('--start_time', type=int, default = None)
    parser.add_argument('--end_time', type=int, default = None)
    parser.add_argument('--interval', type=int, default = 3600*6)

    args = parser.parse_args()

    downloader = Downloader(args.client_id, args.client_secret, args.user_agent)
    downloader.scrape_posts(args.subreddit, args.start_time, args.end_time, args.interval)


"""This module keeps track of current authorization."""
from system_server.jwt import create_jwt, registrant_from_jwt
from datetime import timedelta
import uuid
import logging
from system_server.constants import AUTHORIZATION_AUDIENCE

_log = logging.getLogger(__name__)

AUTHORIZATION_DURATION = timedelta(hours=2)

def authorize_token(
    registration_token: str,
    signing_key: str,
) -> str:
    """Accepts a registration JWT and generates an authorization JWT.

    This function assumes that the token has already been validated by a header authenticator.
    """
    _log.debug(f"Creating authorization token from registration {registration_token}")
    registrant = registrant_from_jwt(token=registration_token, signing_key=signing_key)
    token_id = str(uuid.uuid4())
    return create_jwt(
        signing_key=signing_key,
        duration=AUTHORIZATION_DURATION,
        registrant=registrant,
        audience=AUTHORIZATION_AUDIENCE,
        id=token_id,
    )


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2020/12/11 14:57

import logging
import os
import datetime

class Log():

    def __init__(self):
        self.logger = logging.getLogger()
        self.logger.setLevel(logging.DEBUG)

        self.base_dir = os.path.join(os.path.abspath(os.path.dirname(os.path.dirname(__file__))), 'Logs')

        self.log_file = datetime.datetime.now().strftime("%Y-%m-%d")+".log"

        self.log_name = os.path.join(self.base_dir, self.log_file)

        # #文件输出日志
        self.file_handle = logging.FileHandler(self.log_name,"a",encoding="utf-8")
        self.formatter = logging.Formatter('%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s')
        self.file_handle.setFormatter(self.formatter)
        self.logger.addHandler(self.file_handle)

    def get_log(self):
        return self.logger

    def close_handle(self):
        self.logger.removeHandler(self.file_handle)
        self.file_handle.close()

if __name__ == '__main__':
    log = Log()
    log.get_log()


import pymysql as sql_tr

sql_con = sql_tr.connect(
    user = 'root',
    password = '',
    host= 'localhost',
    database= 'hr_db'
)

cur = sql_con.cursor()
insert_query = '''
INSERT INTO emp_tbl(emp_name,emp_age ,emp_dob, emp_salary)
VALUES(%s,%s,%s,%s)
'''
data_value = [
    ('John Bosco',23,'1999-03-23', 34567.89),
    ('Ronald Roe',44 ,'1934-09-21', 93564),
    ('Jessical Joe',34,'2004-02-15',34444),
    ('Angel Gomez',22,'2021-07-11',372321)
]
cur.executemany(insert_query, data_value)
sql_con.commit()


import math
import xml.etree.ElementTree as ET

from button_tool.py.vtdTools import Weather

def get_score(xml_path, weather_result):
    ego_car_type = None
    veh_car_type = None
    ttc_pivot = None
    ego_speed = 500
    ego_direction = 500
    veh_speed = 500
    veh_acc_target = 500
    veh_lane = 1000
    ttc = 0
    veh_lane_change = 1000
    veh_lane_change_time = 1000
    veh_acc = 1000
    ego_x = 100000
    ego_y = 100000
    veh_x = 100000
    veh_y = 100000

    require_weather = Weather.RAIN
    if require_weather == weather_result:
        weather_score = True
    else:
        weather_score = False
    try:
        item = 1
        score = 0
        distance = 0
        tree = ET.parse(xml_path)
        root = tree.getroot()
        for player in root.iter('Player'):
            for des in player.iter('Description'):
                if des.attrib['Name'] == 'Ego':
                    ego_car_type = des.attrib['Type']
                    for ego_init in player.iter('Init'):
                        for ego_data in ego_init:
                            if ego_data.tag == 'Speed':
                                ego_speed = float(ego_data.attrib['Value'])
                            elif ego_data.tag == 'PosAbsolute':
                                ego_direction = float(ego_data.attrib['Direction'])
                                ego_x = float(ego_data.attrib['X'])
                                ego_y = float(ego_data.attrib['Y'])

                elif des.attrib['Name'] == 'veh_1':
                    veh_car_type = des.attrib['Type']
                    for veh_init in player.iter('Init'):
                        for veh_data in veh_init:
                            if veh_data.tag == 'Speed':
                                veh_speed = float(veh_data.attrib['Value'])
                            elif veh_data.tag == 'PosRelative':
                                if veh_data.attrib['Pivot'] == 'Ego':
                                    distance = float(veh_data.attrib['Distance'])
                                    veh_lane = int(veh_data.attrib['Lane'])
                            elif veh_data.tag == 'PosAbsolute':
                                veh_x = float(veh_data.attrib['X'])
                                veh_y = float(veh_data.attrib['Y'])
                                distance = float(veh_x - ego_x)
                                if -2.825 <= veh_y <= -1.175:
                                    veh_lane = 1

        for player_actions in root.iter('PlayerActions'):
            if player_actions.attrib['Player'] == 'veh_1':
                for speed_change in player_actions.iter('SpeedChange'):
                    veh_acc = float(speed_change.attrib['Rate'])
                    veh_acc_target = float(speed_change.attrib['Target'])
                for ttc_about in player_actions.iter('TTCRelative'):
                    ttc_pivot = ttc_about.attrib['Pivot']
                    ttc = float(ttc_about.attrib['TTC'])
                for lane_change in player_actions.iter('LaneChange'):
                    veh_lane_change = int(lane_change.attrib['Direction'])
                    veh_lane_change_time = float(lane_change.attrib['Time'])

        if ego_speed == 80 / 3.6 and ego_direction == math.radians(5) and ego_car_type == 'CICV_Car':
            score += 1
            xml_score_detail = f'{item}.测试车(Ego)车型为CICV_Car,以80km/h,初始车头方向偏离车道5°,沿直道匀速行驶,且不驶出本车道,得1分;
'
        else:
            xml_score_detail = f'{item}.不满足测试车(Ego)车型为CICV_Car,以80km/h,初始车头方向偏离车道5°,沿直道匀速行驶,且不驶出本车道,不得分;
'
        item += 1
        if 19.99 / 3.6 <= veh_speed <= 30.01 / 3.6 and veh_lane == 1 and 100 <= distance <= 150 and ttc_pivot == 'Ego' and \
                ttc >= 5 and veh_lane_change == -1 and veh_acc_target == 0 and veh_car_type == 'Audi_A3_2009_blue' and \
                veh_lane_change_time == 1 and veh_acc <= 1:
            score += 1
            xml_score_detail = xml_score_detail + f'{item}.障碍车(veh_1)车型为Audi_A3_2009_blue,位于Ego车左侧车道的前方,与Ego车相距100-150m,且当TTC不小于5秒时,障碍车以20-30km/h的初速度开始变道,并切入到自车前方,换道时间为1s,减速度不超过1m/s²,减速至静止,得1分;
'
        else:
            xml_score_detail = xml_score_detail + f'{item}.不满足障碍车(veh_1)车型为Audi_A3_2009_blue,位于Ego车左侧车道的前方,与Ego车相距100-150m,且当TTC不小于5秒时,障碍车以20-30km/h的初速度开始变道,并切入到自车前方,换道时间为1s,减速度不超过1m/s²,减速至静止,不得分;
'
        item += 1
        return score, xml_score_detail, weather_score, item
    except:
        return 0, '场景文件格式错误，场景搭建得0分;
', weather_score, 1

if __name__ == '__main__':
    score = get_score('/home/server/Documents/upload_2.xml', Weather.RAIN)
    print(score)


from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC

def manageStudent_test(driver, url):
    while(True):
        try:#click on Manage Student from the menu
            driver.find_element(By.CSS_SELECTOR, "[data-testid=\"GroupIcon\"]").click()
            break
        except Exception as e:
            pass
    print("In ManageStudent Menu")
    driver.implicitly_wait(20)
    addStudentByForm = driver.find_element(By.CLASS_NAME, "MuiButton-root")
    addStudentByForm.click()
    driver.implicitly_wait(40)
    admissionNo_input = driver.find_element(By.CSS_SELECTOR, 'input[aria-invalid="false"][name="admissionNo"]')
    admissionNo_input.send_keys("102-2023")
    driver.implicitly_wait(30)
    firstName_input = driver.find_element(By.CSS_SELECTOR, 'input[aria-invalid="false"][name="firstName"]')
    firstName_input.send_keys("Jonathan")
    driver.implicitly_wait(50)
    lastName_input = driver.find_element(By.CSS_SELECTOR, 'input[aria-invalid="false"][name="lastName"]')
    lastName_input.send_keys("Mishra")
    driver.implicitly_wait(60)
    fatherName_input = driver.find_element(By.CSS_SELECTOR, 'input[aria-invalid="false"][name="fathersName"]')
    fatherName_input.send_keys("Jhonny")
    driver.implicitly_wait(50)
    mobile_input = driver.find_element(By.CSS_SELECTOR, 'input[aria-invalid="false"][name="mobileNo"]')
    mobile_input.send_keys("3652147896")
    dob_input = driver.find_element(By.CSS_SELECTOR, 'input[aria-invalid="false"][name="dateOfBirth"]')
    dob_input.send_keys("25-08-2000")
    while(True):
        try:
            allDropdownMenu = driver.find_elements(By.CSS_SELECTOR, ".MuiFormControl-root.MuiFormControl-fullWidth.css-1mejovb")
            allDropdownMenu[0].click()
            driver.implicitly_wait(50)
            # class_selection
            driver.find_element(By.XPATH, "//li[@data-value='CLASS 12']").click()
            allDropdownMenu[1].click()
            driver.implicitly_wait(50)
            # divison_selection
            driver.find_element(By.XPATH, "//li[@data-value='B']").click()
            allDropdownMenu[2].click()
            driver.implicitly_wait(50)
            # geneder_selection
            driver.find_element(By.XPATH, "//li[@data-value='M']").click()
            allDropdownMenu[3].click()
            driver.implicitly_wait(50)
            # bus selection
            driver.find_element(By.XPATH, "//li[@data-value='01']").click()
            allDropdownMenu[4].click()
            driver.implicitly_wait(50)
            # payment status selection
            driver.find_element(By.XPATH, "//li[@data-value='Yes']").click()
            allDropdownMenu[5].click()
            driver.implicitly_wait(50)
            # registered selection
            driver.find_element(By.XPATH, "//li[@data-value='Yes']").click()
            allDropdownMenu[7].click()
            driver.implicitly_wait(50)
            # admission status selection
            driver.find_element(By.XPATH, "//li[@data-value='Studying']").click()
            allDropdownMenu[8].click()
            driver.implicitly_wait(50)
            # hostel selection
            driver.find_element(By.XPATH, "//li[@data-value='Hostel']").click()
            # allDropdownMenu[6].click()
            # # area selection
            # driver.find_element(By.XPATH, "//li[@data-value='None']").click()
            break
        except Exception as e:
            pass
    addStudentDetails = driver.find_element(By.CSS_SELECTOR, ".MuiButton-root.MuiButton-text.MuiButton-textPrimary.MuiButton-sizeMedium.MuiButton-textSizeMedium.MuiButton-fullWidth.MuiButtonBase-root")
    addStudentDetails.click()
    confirm_Student_details = driver.find_elements(By.CSS_SELECTOR, ".MuiButton-root.MuiButton-text.MuiButton-textPrimary.MuiButton-sizeMedium.MuiButton-textSizeMedium.MuiButton-fullWidth.MuiButtonBase-root")
    confirm_Student_details_button = confirm_Student_details[1]
    driver.execute_script("arguments[0].scrollIntoView();", confirm_Student_details_button)
    confirm_Student_details_button.click()

    # check student details
    confirm_elements = driver.find_elements(By.CSS_SELECTOR, ".MuiOutlinedInput-input.MuiInputBase-input.Mui-disabled.css-1x5jdmq")
    print("Full Name: ",confirm_elements[0].get_attribute("value"))
    print("Admission No: ",confirm_elements[1].get_attribute("value"))
    print("Class: ",confirm_elements[2].get_attribute("value"))
    print("Division: ",confirm_elements[3].get_attribute("value"))
    print("Gender: ",confirm_elements[4].get_attribute("value"))
    print("Area: ",confirm_elements[5].get_attribute("value"))
    print("Address: ",confirm_elements[6].get_attribute("value"))
    print("Mobile No: ",confirm_elements[7].get_attribute("value"))
    print("Bus ID: ",confirm_elements[8].get_attribute("value"))
    print("Payment Status: ",confirm_elements[9].get_attribute("value"))
    print("Registered: ",confirm_elements[10].get_attribute("value"))