import time

import gobject
import gtk
import logging

from miro import app
from miro import player
from miro.gtcache import gettext as _
from miro import signals
from miro import util
from miro import messages
from miro import displaytext
from miro.plat import resources
from miro.plat import screensaver
from miro.frontends.widgets.gtk.window import Window, WrappedWindow
from miro.frontends.widgets.gtk.widgetset import (Widget, VBox, Label, HBox,
                                                  Alignment, Background,
                                                  DrawingArea, ImageSurface,
                                                  Image, CustomButton)
from miro.frontends.widgets.gtk.persistentwindow import PersistentWindow

BLACK = (0.0, 0.0, 0.0)
WHITE = (1.0, 1.0, 1.0)

# Global VideoWidget object.  We re-use so we can re-use our PersistentWindow
video_widget = None

class ClickableLabel(Widget):
    """This is like a label and reimplements many of the Label things, but
    it's an EventBox with a Label child widget.
    """
    def __init__(self, text, size=0.77, color=WHITE):
        Widget.__init__(self)
        self.set_widget(gtk.EventBox())

        self.label = Label(text)

        self._widget.add(self.label._widget)
        self.label._widget.show()
        self._widget.set_above_child(False)
        self._widget.set_visible_window(False)

        self.set_size(size)
        self.set_color(color)

        self.wrapped_widget_connect('button-release-event', self.on_click)
        self.wrapped_widget_connect('enter-notify-event', self.on_enter_notify)
        self.wrapped_widget_connect('leave-notify-event', self.on_leave_notify)
        self.wrapped_widget_connect('unmap', self.on_unmap)
        self.create_signal('clicked')

    def on_click(self, widget, event):
        self.emit('clicked', event)
        return True

    def on_enter_notify(self, widget, event):
        self._widget.window.set_cursor(gtk.gdk.Cursor(gtk.gdk.HAND1))

    def on_leave_notify(self, widget, event):
        if self._widget.window:
            self._widget.window.set_cursor(None)

    def on_unmap(self, widget):
        if self._widget.window:
            self._widget.window.set_cursor(None)

    def set_size(self, size):
        self.label.set_size(size)

    def set_color(self, color):
        self.label.set_color(color)

    def set_text(self, text):
        self.label.set_text(text)

    def hide(self):
        self.label._widget.hide()

    def show(self):
        self.label._widget.show()

class ClickableImageButton(CustomButton):
    def __init__(self, image_path):
        CustomButton.__init__(self)
        self.image = ImageSurface(Image(image_path))

        self.wrapped_widget_connect('enter-notify-event', self.on_enter_notify)
        self.wrapped_widget_connect('leave-notify-event', self.on_leave_notify)
        self.wrapped_widget_connect('button-release-event', self.on_click)

    def size_request(self, layout):
        return self.image.width, self.image.height

    def draw(self, context, layout):
        self.image.draw(context, 0, 0, self.image.width, self.image.height)

    def on_click(self, widget, event):
        self.emit('clicked', event)
        return True

    def on_enter_notify(self, widget, event):
        self._widget.window.set_cursor(gtk.gdk.Cursor(gtk.gdk.HAND1))

    def on_leave_notify(self, widget, event):
        if self._widget.window:
            self._widget.window.set_cursor(None)

class NullRenderer:
    def __init__(self):
        pass

    def reset(self):
        pass

def make_hidden_cursor():
    pixmap = gtk.gdk.Pixmap(None, 1, 1, 1)
    color = gtk.gdk.Color()
    return gtk.gdk.Cursor(pixmap, pixmap, color, color, 0, 0)

def make_label(text, handler, visible=True):
    if visible:
        lab = ClickableLabel(text)
        lab.connect('clicked', handler)
        return lab

    # if the widget isn't visible, then we stick in an empty string--we just
    # need a placeholder so that things don't move around when the item state
    # changes.
    lab = Label("")
    lab.set_size(0.77)
    return lab

def make_image_button(image_path, handler):
    b = ClickableImageButton(resources.path(image_path))
    b.connect('clicked', handler)
    return b

def _align_left(widget, top_pad=0, bottom_pad=0, left_pad=0, right_pad=0):
    """Align left and pad."""
    alignment = Alignment(0, 0, 0, 1)
    alignment.set_padding(top_pad, bottom_pad, left_pad, right_pad)
    alignment.add(widget)
    return alignment

def _align_right(widget, top_pad=0, bottom_pad=0, left_pad=0, right_pad=0):
    """Align right and pad."""
    alignment = Alignment(1, 0, 0, 1)
    alignment.set_padding(top_pad, bottom_pad, left_pad, right_pad)
    alignment.add(widget)
    return alignment

def _align_center(widget, top_pad=0, bottom_pad=0, left_pad=0, right_pad=0):
    """Align center (horizontally) and pad."""
    alignment = Alignment(0.5, 0, 0, 1)
    alignment.set_padding(top_pad, bottom_pad, left_pad, right_pad)
    alignment.add(widget)
    return alignment

def _align_middle(widget, top_pad=0, bottom_pad=0, left_pad=0, right_pad=0):
    """Align center (vertically) and pad."""
    alignment = Alignment(0, 0.5, 0, 0)
    alignment.set_padding(top_pad, bottom_pad, left_pad, right_pad)
    alignment.add(widget)
    return alignment

# Couple of utility functions to grab GTK widgets out of the widget tree for
# fullscreen code
def _videobox_widget():
    return app.widgetapp.window.videobox._widget

def _window():
    """Returns the window used for playback.  This is either the main window
    or the detached window.
    """
    if app.playback_manager.detached_window:
        return app.playback_manager.detached_window._window
    return app.widgetapp.window._window

class VideoOverlay(Window):
    def __init__(self):
        Window.__init__(self, 'Miro Video Overlay')
        self._window.set_transient_for(_window())
        self.vbox = VBox()
        self.set_content_widget(self.vbox)

    def _make_gtk_window(self):
        return WrappedWindow(gtk.WINDOW_POPUP)

    def position_on_screen(self):
        window = self._window
        parent_window = window.get_transient_for()
        screen = parent_window.get_screen()
        monitor = screen.get_monitor_at_window(parent_window.window)
        screen_rect = screen.get_monitor_geometry(monitor)
        my_width, my_height = self.vbox.get_size_request()
        window.set_default_size(my_width, my_height)
        window.resize(screen_rect.width, my_height)
        window.move(screen_rect.x, screen_rect.y + screen_rect.height -
                my_height)

class VideoWidget(Widget):
    def __init__(self, renderer):
        Widget.__init__(self)
        self.set_widget(PersistentWindow())
        self._widget.set_double_buffered(False)
        self._widget.add_events(gtk.gdk.POINTER_MOTION_MASK)
        self._widget.add_events(gtk.gdk.BUTTON_PRESS_MASK)
        renderer.set_widget(self._widget)

    def destroy(self):
        self._widget.destroy()

class Divider(DrawingArea):
    def size_request(self, layout):
        return (1, 25)

    def draw(self, context, layout):
        context.set_line_width(1)
        context.set_color((46.0 / 255.0, 46.0 / 255.0, 46.0 / 255.0))
        context.move_to(0, 0)
        context.rel_line_to(0, context.height)
        context.stroke()

class VideoDetailsWidget(Background):
    def __init__(self):
        Background.__init__(self)
        self.item_info = None
        self.rebuild_video_details()
        self._delete_link = None

    def rebuild_video_details(self):
        # this removes the child widget if there is one
        self.remove()

        if not self.item_info:
            self.add(HBox())
            return

        info = self.item_info

        outer_hbox = HBox(5)

        if not info.is_external:
            if info.expiration_date is not None:
                text = displaytext.expiration_date(info.expiration_date)
                self._expiration_label = Label(text)
                self._expiration_label.set_size(0.77)
                self._expiration_label.set_color((152.0 / 255.0, 152.0 / 255.0, 152.0 / 255.0))
                outer_hbox.pack_start(_align_middle(self._expiration_label))
                outer_hbox.pack_start(_align_middle(Divider(), top_pad=3, bottom_pad=3, left_pad=5, right_pad=5))

            lab = make_label(_("Keep"), self.handle_keep, info.expiration_date is not None)
            outer_hbox.pack_start(_align_middle(lab))
            outer_hbox.pack_start(_align_middle(Divider(), top_pad=3, bottom_pad=3, left_pad=5, right_pad=5))

        self._subtitles_link = make_label(_("Subtitles"),
                                          self.handle_subtitles)
        outer_hbox.pack_start(_align_middle(self._subtitles_link))
        subtitles_image = make_image_button('images/subtitles_down.png',
                                            self.handle_subtitles)
        outer_hbox.pack_start(_align_middle(subtitles_image))

        outer_hbox.pack_start(_align_middle(Divider(), top_pad=3, bottom_pad=3, left_pad=5, right_pad=5))

        self._delete_link = make_label(_("Delete"), self.handle_delete)
        outer_hbox.pack_start(_align_middle(self._delete_link))

        if not info.is_external:
            outer_hbox.pack_start(_align_middle(Divider(), top_pad=3, bottom_pad=3, left_pad=5, right_pad=5))

            self._share_link = make_label(_("Share"), self.handle_share,
                                          info.has_sharable_url)
            outer_hbox.pack_start(_align_middle(self._share_link))
            outer_hbox.pack_start(_align_middle(Divider(), top_pad=3, bottom_pad=3, left_pad=5, right_pad=5))

            if info.commentslink:
                self._permalink_link = make_label(_("Comments"),
                                                  self.handle_commentslink,
                                                  info.commentslink)
            else:
                self._permalink_link = make_label(_("Permalink"),
                                                  self.handle_permalink,
                                                  info.permalink)
            outer_hbox.pack_start(_align_middle(self._permalink_link))

        outer_hbox.pack_start(_align_middle(Divider(), top_pad=3, bottom_pad=3, left_pad=5, right_pad=5))

        if app.playback_manager.is_fullscreen:
            fullscreen_link = make_label(_("Exit fullscreen"),
                                         self.handle_fullscreen)
            outer_hbox.pack_start(_align_middle(fullscreen_link))
            fullscreen_image = make_image_button('images/fullscreen_exit.png',
                                                 self.handle_fullscreen)
            outer_hbox.pack_start(_align_middle(fullscreen_image))
        else:
            fullscreen_link = make_label(_("Fullscreen"),
                                         self.handle_fullscreen)
            outer_hbox.pack_start(_align_middle(fullscreen_link))
            fullscreen_image = make_image_button('images/fullscreen_enter.png',
                                                 self.handle_fullscreen)
            outer_hbox.pack_start(_align_middle(fullscreen_image))

        if app.playback_manager.detached_window is not None:
            popin_link = make_label(_("Pop-in"), self.handle_popin_popout)
            outer_hbox.pack_start(_align_middle(popin_link))
            popin_image = make_image_button('images/popin.png',
                                            self.handle_popin_popout)
            outer_hbox.pack_start(_align_middle(popin_image))
        else:
            popout_link = make_label(_("Pop-out"), self.handle_popin_popout)
            outer_hbox.pack_start(_align_middle(popout_link))
            popout_image = make_image_button('images/popout.png',
                                             self.handle_popin_popout)
            outer_hbox.pack_start(_align_middle(popout_image))

        self.add(_align_right(outer_hbox, left_pad=15, right_pad=15))

    def hide(self):
        self._widget.hide()

    def show(self):
        self._widget.show()

    def handle_fullscreen(self, widget, event=None):
        app.playback_manager.toggle_fullscreen()

    def handle_popin_popout(self, widget, event=None):
        if app.playback_manager.is_fullscreen:
            app.playback_manager.exit_fullscreen()
        app.playback_manager.toggle_detached_mode()

    def handle_keep(self, widget, event):
        messages.KeepVideo(self.item_info.id).send_to_backend()
        self._widget.window.set_cursor(None)

    def handle_delete(self, widget, event):
        item_info = self.item_info
        self.reset()
        app.widgetapp.remove_items([item_info])

    def handle_subtitles(self, widget, event):
        tracks = []
        menu = gtk.Menu()

        tracks = app.video_renderer.get_subtitle_tracks()

        if len(tracks) == 0:
            child = gtk.MenuItem(_("None Available"))
            child.set_sensitive(False)
            child.show()
            menu.append(child)
        else:
            enabled_track = app.video_renderer.get_enabled_subtitle_track()

            first_child = None
            for i, lang in tracks:
                child = gtk.RadioMenuItem(first_child, lang)
                if enabled_track == i:
                    child.set_active(True)
                child.connect('activate', self.handle_subtitle_change, i)
                child.show()
                menu.append(child)
                if first_child == None:
                    first_child = child

            sep = gtk.SeparatorMenuItem()
            sep.show()
            menu.append(sep)

            child = gtk.RadioMenuItem(first_child, _("Disable Subtitles"))
            if enabled_track == -1:
                child.set_active(True)
            child.connect('activate', self.handle_disable_subtitles)
            child.show()
            menu.append(child)

        sep = gtk.SeparatorMenuItem()
        sep.show()
        menu.append(sep)

        child = gtk.MenuItem(_("Select a Subtitles file..."))
        child.set_sensitive(True)
        child.connect('activate', self.handle_select_subtitle_file)
        child.show()
        menu.append(child)

        menu.popup(None, None, None, event.button, event.time)

    def handle_disable_subtitles(self, widget):
        if widget.active:
            app.video_renderer.disable_subtitles()
            app.widgetapp.window.on_playback_change(app.playback_manager)

    def handle_subtitle_change(self, widget, index):
        if widget.active:
            app.video_renderer.enable_subtitle_track(index)
            app.widgetapp.window.on_playback_change(app.playback_manager)

    def handle_select_subtitle_file(self, widget):
        app.playback_manager.open_subtitle_file()

    def handle_commentslink(self, widget, event):
        app.widgetapp.open_url(self.item_info.commentslink)

    def handle_share(self, widget, event):
        app.widgetapp.share_item(self.item_info)

    def handle_permalink(self, widget, event):
        app.widgetapp.open_url(self.item_info.permalink)

    def update_info(self, item_info):
        self.item_info = item_info
        self.rebuild_video_details()

    def set_video_details(self, item_info):
        """This gets called when the item is set to play.  It should make
        no assumptions about the state of the video details prior to being
        called.
        """
        self.update_info(item_info)

    def draw(self, context, layout):
        context.set_color(BLACK)
        context.rectangle(0, 0, context.width, context.height)
        context.fill()
        context.set_color((46.0 / 255.0, 46.0 / 255.0, 46.0 / 255.0))
        context.set_line_width(1)
        context.move_to(0, 0)
        context.rel_line_to(context.width, 0)
        context.stroke()

    def is_opaque(self):
        return True

    def reset(self):
        if self._delete_link:
            self._delete_link.on_leave_notify(None, None)

class VideoPlayer(player.Player, VBox):
    """Video renderer widget.

    Note: ``app.video_renderer`` must be initialized before instantiating this
    class.  If no renderers can be found, set ``app.video_renderer`` to ``None``.
    """
    HIDE_CONTROLS_TIMEOUT = 2000

    def __init__(self):
        global video_widget
        player.Player.__init__(self)
        VBox.__init__(self)
        if app.video_renderer is not None:
            self.renderer = app.video_renderer
        else:
            self.renderer = NullRenderer()

        self.overlay = None

        if video_widget is None:
            video_widget = VideoWidget(self.renderer)
        self._video_widget = video_widget
        self.pack_start(self._video_widget, expand=True)

        self._video_details = VideoDetailsWidget()
        self.pack_start(self._video_details)

        self.hide_controls_timeout = None
        self.motion_handler = None
        self.videobox_motion_handler = None
        self.hidden_cursor = make_hidden_cursor()
        # piggyback on the TrackItemsManually message that playback.py sends.
        app.info_updater.item_changed_callbacks.add('manual', 'playback-list',
                self._on_items_changed)
        self._item_id = None

        self._video_widget.wrapped_widget_connect('button-press-event', self.on_button_press)

    def teardown(self):
        self.renderer.reset()
        app.info_updater.item_changed_callbacks.remove('manual',
                'playback-list', self._on_items_changed)
        self._items_changed_callback = None
        self.remove(self._video_widget)

    def _on_items_changed(self, message):
        for item_info in message.changed:
            if item_info.id == self._item_id:
                self._video_details.update_info(item_info)
                break

    def update_for_presentation_mode(self, mode):
        pass

    def set_item(self, item_info, success_callback, error_callback):
        self._video_details.set_video_details(item_info)
        self.renderer.select_file(item_info, success_callback, error_callback)
        self._item_id = item_info.id

    def get_elapsed_playback_time(self):
        return self.renderer.get_current_time()

    def get_total_playback_time(self):
        return self.renderer.get_duration()

    def set_volume(self, volume):
        self.renderer.set_volume(volume)

    def play(self):
        self.renderer.play()
        # do this to trigger the overlay showing up for a smidge
        self.on_mouse_motion(None, None)

    def play_from_time(self, resume_time=0):
        # Note: This overrides player.Player's version of play_from_time, but
        # this one seeks directly rather than fiddling with
        # total_playback_time.
        self.seek_to_time(resume_time)
        self.play()

    def pause(self):
        self.renderer.pause()

    def stop(self, will_play_another=False):
        self._video_details.reset()
        self.renderer.stop()

    def set_playback_rate(self, rate):
        self.renderer.set_rate(rate)

    def seek_to(self, position):
        duration = self.get_total_playback_time()
        if duration is None:
            return
        time = duration * position
        self.seek_to_time(time)

    def seek_to_time(self, time_pos):
        self.renderer.set_current_time(time_pos)

    def enter_fullscreen(self):
        self.screensaver_manager = screensaver.create_manager()
        if self.screensaver_manager is not None:
            self.screensaver_manager.disable()
        self.rebuild_video_details()
        self._make_overlay()
        self.motion_handler = self.wrapped_widget_connect(
                'motion-notify-event', self.on_mouse_motion)
        self.videobox_motion_handler = self.overlay._window.connect(
                'motion-notify-event', self.on_mouse_motion)
        if not app.playback_manager.detached_window:
            app.widgetapp.window.menubar.hide()
        self.schedule_hide_controls(self.HIDE_CONTROLS_TIMEOUT)
        # make sure all hide() calls go through, otherwise we get the wrong
        # size on windows (#10810)
        while gtk.events_pending():
            gtk.main_iteration()
        _window().fullscreen()

    def _make_overlay(self):
        main_window = app.widgetapp.window
        main_window.main_vbox.remove(main_window.controls_hbox)
        self.overlay = VideoOverlay()
        self.remove(self._video_details)
        self.overlay.vbox.pack_start(self._video_details)
        self.overlay.vbox.pack_start(main_window.controls_hbox)
        self.overlay.position_on_screen()
        self.overlay.show()

    def _destroy_overlay(self):
        main_window = app.widgetapp.window
        self.overlay.vbox.remove(self._video_details)
        self.overlay.vbox.remove(main_window.controls_hbox)
        self.pack_start(self._video_details)
        main_window.main_vbox.pack_start(main_window.controls_hbox)

        self.overlay.destroy()
        self.overlay = None

    def rebuild_video_details(self):
        self._video_details.rebuild_video_details()

    def prepare_switch_to_attached_playback(self):
        gobject.timeout_add(0, self.rebuild_video_details)

    def prepare_switch_to_detached_playback(self):
        gobject.timeout_add(0, self.rebuild_video_details)

    def on_button_press(self, widget, event):
        if event.type == gtk.gdk._2BUTTON_PRESS:
            app.playback_manager.toggle_fullscreen()
            return True
        return False

    def on_mouse_motion(self, widget, event):
        if not self.overlay:
            return
        if not self.overlay.is_visible():
            show_it_all = False

            if event is None:
                show_it_all = True
            else:
                # figures out the monitor that miro is fullscreened on and
                # gets the monitor geometry for that.
                if app.playback_manager.detached_window is not None:
                    gtkwindow = app.playback_manager.detached_window._window
                else:
                    gtkwindow = app.widgetapp.window._window
                gdkwindow = gtkwindow.window
                screen = gtkwindow.get_screen()

                monitor = screen.get_monitor_at_window(gdkwindow)
                monitor_geom = screen.get_monitor_geometry(monitor)
                if event.y > monitor_geom.height - 200:
                    show_it_all = True

            if show_it_all:
                self.show_controls()
            else:
                self.show_mouse()
            self.schedule_hide_controls(self.HIDE_CONTROLS_TIMEOUT)
        else:
            self.last_motion_time = time.time()

    def show_mouse(self):
        _window().window.set_cursor(None)

    def show_controls(self):
        logging.info("show_controls")
        self.show_mouse()
        self.overlay.show()

    def hide_controls(self):
        _window().window.set_cursor(self.hidden_cursor)
        if self.overlay and self.overlay.is_visible():
            self.overlay.close()

    def on_hide_controls_timeout(self):
        # Check if the mouse moved before the timeout
        time_since_motion = int((time.time() - self.last_motion_time) * 1000)
        timeout_left = self.HIDE_CONTROLS_TIMEOUT - time_since_motion
        if timeout_left <= 0:
            self.hide_controls()
            self.hide_controls_timeout = None
        else:
            self.schedule_hide_controls(timeout_left)

    def cancel_hide_controls(self):
        if self.hide_controls_timeout is not None:
            gobject.source_remove(self.hide_controls_timeout)

    def schedule_hide_controls(self, time):
        if self.hide_controls_timeout is not None:
            gobject.source_remove(self.hide_controls_timeout)
        self.hide_controls_timeout = gobject.timeout_add(time,
                self.on_hide_controls_timeout)
        self.last_motion_time = 0

    def exit_fullscreen(self):
        if self.screensaver_manager is not None:
            self.screensaver_manager.enable()
            self.screensaver_manager = None
        app.widgetapp.window.menubar.show()
        self.rebuild_video_details()
        self._video_details.show()
        self._destroy_overlay()
        _window().unfullscreen()
        self._widget.disconnect(self.motion_handler)
        self.cancel_hide_controls()
        _window().window.set_cursor(None)

    def select_subtitle_file(self, sub_path, handle_successful_select):
        app.video_renderer.select_subtitle_file(
            app.playback_manager.get_playing_item(),
            sub_path,
            handle_successful_select)

    def select_subtitle_encoding(self, encoding):
        app.video_renderer.select_subtitle_encoding(encoding)


from flask import Blueprint, request, jsonify
from flask_cors import cross_origin
from database_functions.db_connection.connection import connection
from database_functions.account.token_auth_flow import refresh_token
from database_functions.groups.insertion_functions import insert_into_group_table, insert_into_pending_requests_table
from database_functions.logs.recentLogs import insert_into_recent_table
from database_functions.groups.querying_functions import get_status_for_group, get_group_title, get_group_pending_users \
    , get_group_pending_state_machine
from database_functions.groups.updation_functions import update_group_status, add_new_pending_users_group,\
    update_pending_state_machine, resolve_admin_approval
from ast import literal_eval
from time import time

addUsersGroupApi = Blueprint('addUsersGroupApi', __name__)

# API to add users to a group
@addUsersGroupApi.route('/group/addUsersGroup', methods=['POST'])
@cross_origin()
def add_users_to_group():
    try:
        user_name = request.json['user_name']
        refresh_token(connection(), user_name)
        users = request.json['users']
        users = list(set(users))  # Avoid adding same users multiple times to the groups

        group_id = request.json['group_id']
        group_title = get_group_title(connection(), group_id)

        # updating list of pending users
        pending_users = get_group_pending_users(connection(), group_id)
        new_set_users = literal_eval(pending_users)
        new_set_users = list(set(new_set_users + users))
        add_new_pending_users_group(connection(), group_id, str(new_set_users))

        response = {'success': [], 'fail': []}

        if not group_title:
            return jsonify(False)
        for user in users:
            try:  # resolving admin approvals
                group_admin = request.json['group_admin']
                resolve_admin_approval(connection(), group_admin, "add", user, group_id)
            except:
                pass

            #  if user has rejected the group, or exited, or has been removed, update the status to pending
            if get_status_for_group(connection(), group_id, user, "rejected") or \
                    get_status_for_group(connection(), group_id, user, "removed") or \
                    get_status_for_group(connection(), group_id, user, "exited") or \
                    get_status_for_group(connection(), group_id, user, "pending") or \
                    get_status_for_group(connection(), group_id, user, "awaiting"):
                if get_status_for_group(connection(), group_id, user, "rejected") and \
                        get_group_pending_state_machine(connection(), user, group_id) >= 4:
                    # user has rejected the group too many times, can't be added
                    response['fail'].append(user)
                else:  # successfully add the user to the group
                    update_group_status(connection(), group_id, user, "pending")
                    response['success'].append(user)
            #  if user entry does not exist in the group, then add a new entry
            elif not get_status_for_group(connection(), group_id, user, "accepted"):
                response['success'].append(user)
                insert_into_pending_requests_table(connection(), group_id, user, "pending", 0)
        # adding transaction to logs

        if response['success']:
            message = "You added a few users to the group " + group_title + ". The users are: "
            message_description = response['success']
            insert_into_recent_table(connection(), user_name, str(time()), "6:Added users to group " + group_title,
                                     message + str(message_description))

        return jsonify(response)
    except:
        return jsonify(False)


import sys
sys.path.insert(0, r'D:\Projects\aaadoc')

from statistics import mean, median
from collections import Counter
from src.utils.utils import get_files_in_from_directory
from src.loaders.NvdLoader import NvdLoader
from src.loaders.OsvLoader import OsvLoader
from src.loaders.OmniLoader import OmniLoader
import json
import dateutil.parser as parser
import pandas as pd
import matplotlib.pyplot as plt
import random
import regex as re

input_data_location = r'D:\Projects\aaa\results\checkpoints_fixMapper'
keywords_regexes_path = r'D:\Projects\aaadoc\src\2_phrase_search\bigquery\keywords_v2.csv'
vfm_classified_commits = r'D:\Projects\aaadoc\src\recalls\data\sap_result_training_excluded.json'
security_relevant_commits_file = r'src\recalls\data\security_relevant_commits.csv'
security_relevant_commits_info_file = r'src\recalls\data\security_relevant_commits_info.json'

fb_predictions_all = r'D:\Projects\aaadoc\src\recalls\data\prediction_all.json'
fb_predictions_npm = r'D:\Projects\aaadoc\src\recalls\data\prediction_npm.json'
fb_predictions_pypi = r'D:\Projects\aaadoc\src\recalls\data\prediction_pypi.json'
fb_predictions_mvn = r'D:\Projects\aaadoc\src\recalls\data\prediction_mvn.json'

commits_info_data = {}
commits_data = {}
security_label_keywords  = ['secur', 'vulnerab', 'exploit']

phrases_to_ids = {}

# General
def _get_ref_date(ref):
    ref_data = ref['data_obj']
    if ref['reference_type'] == 'commit':
        date1 = parser.parse(ref_data['commit']['committer']['date'])
        date2 = parser.parse(ref_data['commit']['author']['date'])
        first = date1 if date1 < date2 else date2
        return first

    if ref['reference_type'] == 'issue':
        date2 = parser.parse(ref_data['created_at'])
        return date2

    if ref['reference_type'] == 'compare':
        dates = []
        for x in ref_data['commits']:
            dates.append(parser.parse(x['commit']['author']['date']))
            dates.append(parser.parse(x['commit']['committer']['date']))
        first = min(dates)
        return first

# General
def _get_ref_id(ref):
    return f'{ref["repo_owner"]}/{ref["repo_name"]}/{ref["reference_type"]}/{ref["reference_value"]}'

# General
def _commit_size(ref):
    global commits_info_data
    if ref["reference_type"] == 'commit':
        commit_id = f'{ref["repo_owner"]}/{ref["repo_name"]}/{ref["reference_value"]}'
        if commit_id in commits_info_data:
            return commits_info_data[commit_id]['commit_info']['stats']['total']
        else:
            return None
    else:
        return None

def _commit_size_by_commit_id(commit_id):
    global commits_info_data
    if commit_id in commits_info_data:
        return commits_info_data[commit_id]['commit_info']['stats']['total']
    else:
        return None

def _get_matched_ref_object(ref, ref_date):
    ref_id = _get_ref_id(ref)
    ref_commit_size = _commit_size(ref)
    repo_full_name = f'{ref["repo_owner"]}/{ref["repo_name"]}'
    matched_ref_obj = {
        'ref_id':ref_id,
        'ref_repo_full_name':repo_full_name,
        'ref_commit_size':ref_commit_size,
        'ref_date': ref_date
    }
    return matched_ref_obj

def _get_matched_ref_object_by_commit_id(commit_id, first):
    segments = commit_id.split('/')
    ref_id = f'{segments[0]}/{segments[1]}/commit/{segments[2]}'
    ref_commit_size = _commit_size_by_commit_id(commit_id)
    repo_full_name = f'{segments[0]}/{segments[1]}'
    matched_ref_obj = {
                    'ref_id':ref_id,
                    'ref_repo_full_name':repo_full_name,
                    'ref_commit_size':ref_commit_size,
                    'ref_date': first
                }

    return matched_ref_obj

# First commit helper
def _get_commits_from_issue(repo_owner, repo_name, issue_data):
    global commits_data

    repo_full_name = f'{repo_owner}/{repo_name}'
    issue_commits = []

    timeline_commits = _extract_commits_from_timeline(repo_owner, repo_name, issue_data['timeline_data'])
    issue_commits += [f'{repo_owner}/{repo_name}/{c}' for c in timeline_commits]
    if 'pull_request' in issue_data:
        if 'pull_request_commits' in issue_data:
            pr_commits = issue_data['pull_request_commits']
            for c in pr_commits:
                c_id = f'{repo_owner}/{repo_name}/{c["sha"]}'
                issue_commits.append(c_id)
    else:
        cross_referenced_issues = [x['source']['issue'] for x in issue_data['timeline_data'] if x['event']=='cross-referenced']
        cross_referenced_events_in_the_same_repo = [x for x in cross_referenced_issues if x['repository']['full_name']==repo_full_name]
        for referenced_issue in cross_referenced_events_in_the_same_repo:
            try:
                timeline_commits =  _extract_commits_from_timeline(repo_owner, repo_name, referenced_issue['timeline_data'])
                issue_commits += [f'{repo_owner}/{repo_name}/{c}' for c in timeline_commits]
                if 'pull_request' in referenced_issue:
                    if 'pull_request_commits' in referenced_issue:
                        pr_commits = referenced_issue['pull_request_commits']
                        for c in pr_commits:
                            c_id = f'{repo_owner}/{repo_name}/{c["sha"]}'
                            issue_commits.append(c_id)
            except:
                print('Failed to process ref issue')

    commits = []
    for x in list(set(issue_commits)):
        if x in commits_data:
            commits.append(commits_data[x])
    return commits

# First commit helper
def _extract_commits_from_timeline(repo_owner, repo_name, timeline):
    try:
        result = []
        referenced_commits = [x for x in timeline if 'commit_id' in x and x['commit_id'] != None]
        for c in referenced_commits:
            if 'html_url' in c:
                seg = c['html_url'].split('/')
                ref_repo_owner = seg[3]
                ref_repo_name = seg[4]
            else:
                seg = c['commit_url'].split('/')
                ref_repo_owner = seg[4]
                ref_repo_name = seg[5]

            if repo_owner == ref_repo_owner and repo_name == ref_repo_name:
                result.append(c['commit_id'])
        return result
    except:
        return []

# Security phrases helper
def _safe_add(text, stuff):
    return text + ' ' + stuff if stuff else text

# Security phrases helper
def _contains_secuirty_related_phrases(ref, regexes, report_id):
    text = ''
    asd = ref['data_obj']
    if ref['reference_type'] == 'commit':
        text = _safe_add(text, asd['commit']['message'])
    elif ref['reference_type'] == 'issue':
        text = _safe_add(text, asd['title'])
        text = _safe_add(text, asd['body'])
        for t in asd['timeline_data']:
            if t['event']=='renamed':
                text = _safe_add(text, t['rename']['from'])
                text = _safe_add(text, t['rename']['to'])
            elif t['event']=='commented':
                text = _safe_add(text, t['body'])
            elif t['event']=='committed':
                text = _safe_add(text, t['message'])

        if 'pull_request_data' in asd:
            text = _safe_add(text, asd['pull_request_data']['title'])
            text = _safe_add(text, asd['pull_request_data']['body'])
        if 'pull_request_commits' in asd:
            for c in asd['pull_request_commits']:
                text = _safe_add(text, c['commit']['message'])
        if 'pull_request_comments' in asd:
            for c in asd['pull_request_comments']:
                text = _safe_add(text, c['body'])
    elif ref['reference_type'] == 'compare':
        for c in asd['commits']:
            text = _safe_add(text, c['commit']['message'])

    global phrases_to_ids
    found_phrases = False
    if text:
        for req in regexes:
            if regexes[req].search(text):
                if req not in phrases_to_ids:
                    phrases_to_ids[req] = set()
                phrases_to_ids[req].add(report_id)
                found_phrases = True

    return found_phrases

# Security labels helper
def _security_related_label(label_name):
    for keyword in security_label_keywords:
        if keyword in label_name:
            return True
    return False

# Security labels helper
def _get_labelling_date(issue):
    if 'labels' not in issue:
        return None

    label_names = [xx['name'].lower() for xx in issue['labels']]
    has_security_ralted_label = False
    for label_name in label_names:
        if _security_related_label(label_name):
            has_security_ralted_label = True
            break

    if not has_security_ralted_label:
        return None

    labelled_events = [evt for evt in issue['timeline_data'] if evt['event']=='labeled' and _security_related_label(evt['label']['name'].lower())]
    actual_labelling_date = None
    if len(labelled_events) == 0:
        actual_labelling_date = parser.parse(issue['created_at'])
    else:
        labelling_dates = [parser.parse(evt['created_at']) for evt in labelled_events]
        actual_labelling_date = min(labelling_dates)

    return actual_labelling_date

def load_creation_date(data, omni):
    for x in data.keys():
        dates = omni.publish_date_of_report(x)
        dates = [x['publish_date'] for x in dates]
        first = min(dates)
        if first.year < 2017:
            continue

        data[x]['creation_date'] = first

def load_ecosystem(data, omni):
    for x in data.keys():
        asd = omni.publish_date_of_report(x)
        asd = [x['publish_date'] for x in asd]
        first = min(asd)
        if first.year < 2017:
            continue

        ecosystems = list(set([xx.lower() for xx in omni.ecosystems_of_a_report(x)]))
        data[x]['ecosystem'] = ecosystems

def load_severity(data, omni):
    for x in data.keys():
        asd = omni.publish_date_of_report(x)
        asd = [x['publish_date'] for x in asd]
        first = min(asd)
        if first.year < 2017:
            continue

        severity = omni.highest_severity(x)
        data[x]['severity'] = severity

def load_repositories(data, references_data):
    for x in data.keys():
        if x not in references_data:
            data[x]['repositories'] = []
            continue

        repos = set()
        for ref in references_data[x]:
            repos.add(f'{ref["repo_owner"]}/{ref["repo_name"]}')

        data[x]['repositories'] = list(repos)

def load_all_refs(data, references_data):
    for x in data.keys():
        if x not in references_data:
            data[x]['all_refs'] = None
            continue

        dates = []
        for ref in references_data[x]:
            ref_date = _get_ref_date(ref)
            if ref_date != None:
                matched_ref_obj = _get_matched_ref_object(ref, ref_date)
                dates.append(matched_ref_obj)

        if len(dates) == 0:
            data[x]['all_refs'] = None
        else:
            data[x]['all_refs'] = dates

def load_first_ref(data, references_data):
    for x in data.keys():
        if x not in references_data:
            data[x]['first_ref'] = None
            continue

        dates = {}
        for ref in references_data[x]:
            ref_date = _get_ref_date(ref)
            if ref_date != None:
                repo_full_name = f'{ref["repo_owner"]}/{ref["repo_name"]}'
                if repo_full_name not in dates:
                    dates[repo_full_name]= []
                dates[repo_full_name].append(ref_date)

        if len(dates) == 0:
            data[x]['first_ref'] = None
        else:
            min_dates = [(k,min(v)) for k,v in dates.items()]
            data[x]['first_ref'] = min_dates

def load_first_issue(data, references_data):
    for x in data.keys():
        if x not in references_data:
            data[x]['first_issue'] = None
            continue

        dates = {}
        for ref in references_data[x]:
            if ref['reference_type'] != 'issue':
                continue

            ref_date = _get_ref_date(ref)
            if ref_date != None:
                repo_full_name = f'{ref["repo_owner"]}/{ref["repo_name"]}'
                if repo_full_name not in dates:
                    dates[repo_full_name]= []
                dates[repo_full_name].append(ref_date)

        if len(dates) == 0:
            data[x]['first_issue'] = None
        else:
            min_dates = [(k,min(v)) for k,v in dates.items()]
            data[x]['first_issue'] = min_dates

def load_first_commit(data, references_data):
    for x in data.keys():
        if x not in references_data:
            data[x]['first_commit'] = None
            continue

        dates = {}
        for ref in references_data[x]:
            if ref['reference_type'] == 'issue':
                commits = _get_commits_from_issue(ref['repo_owner'], ref['repo_name'], ref['data_obj'])
                if commits == None or len(commits) ==0:
                    continue

                min_date = parser.parse(commits[0]['commit']['committer']['date'])
                for c in commits:
                    commiter_date = parser.parse(c['commit']['committer']['date'])
                    author_date = parser.parse(c['commit']['author']['date'])
                    if commiter_date < min_date:
                        min_date = commiter_date
                    if author_date < min_date:
                        min_date = author_date

                repo_full_name = f'{ref["repo_owner"]}/{ref["repo_name"]}'
                if repo_full_name not in dates:
                    dates[repo_full_name]= []
                dates[repo_full_name].append(min_date)

            if ref['reference_type'] == 'commit' or ref['reference_type'] == 'compare':
                ref_date = _get_ref_date(ref)
                if ref_date != None:
                    repo_full_name = f'{ref["repo_owner"]}/{ref["repo_name"]}'
                    if repo_full_name not in dates:
                        dates[repo_full_name]= []
                    dates[repo_full_name].append(ref_date)

        if len(dates) == 0:
            data[x]['first_commit'] = None
        else:
            min_dates = [(k,min(v)) for k,v in dates.items()]
            data[x]['first_commit'] = min_dates

def load_first_sec_phrase(data, references_data):
    with open(keywords_regexes_path, 'r') as f:
        regexes = {x.strip(): re.compile(x.strip()) for x in f.readlines()}

    for x in data.keys():
        if x not in references_data:
            data[x]['first_sec_phrase'] = None
            continue

        dates = []
        for ref in references_data[x]:
            if not _contains_secuirty_related_phrases(ref, regexes, x):
                continue

            ref_date = _get_ref_date(ref)
            if ref_date != None:
                matched_ref_obj = _get_matched_ref_object(ref, ref_date)
                dates.append(matched_ref_obj)

        if len(dates) == 0:
            data[x]['first_sec_phrase'] = None
        else:
            data[x]['first_sec_phrase'] = dates

def load_first_sec_label(data, references_data):
    for x in data.keys():
        if x not in references_data:
            data[x]['first_sec_label'] = None
            continue

        dates = []
        for ref in references_data[x]:
            if ref['reference_type'] == 'issue':
                ref_date  = _get_labelling_date(ref['data_obj'])
                if ref_date != None:
                    matched_ref_obj = _get_matched_ref_object(ref, ref_date)
                    dates.append(matched_ref_obj)

        if len(dates) == 0:
            data[x]['first_sec_label'] = None
        else:
            data[x]['first_sec_label'] = dates

def load_fb_commit_classification(data):
    global commits_info_data

    predictions = {}
    prediction_files = [
        fb_predictions_all,
        fb_predictions_npm,
        fb_predictions_pypi,
        fb_predictions_mvn,
    ]
    for prediction_file in prediction_files:
        with open(prediction_file, 'r') as f:
            temp = json.load(f)
            for x in temp:
                if x['actual_value']:
                    if x['commit_id'] not in predictions:
                        predictions[x['commit_id']] = x['prediction'] == 'True'
                    else:
                        if not predictions[x['commit_id']]:
                            predictions[x['commit_id']] = x['prediction'] == 'True'

    sighted_vulnerabilities = set()
    per_vulnerability = {}
    for commit_id in predictions:
        if commit_id not in commits_info_data:
            continue

        commit = commits_info_data[commit_id]['commit_info']
        sighted_vulnerabilities.update(commits_info_data[commit_id]['report_ids'])

        if predictions[commit_id]:
            date1 = parser.parse(commit['commit']['committer']['date'])
            date2 = parser.parse(commit['commit']['author']['date'])
            first = date1 if date1 < date2 else date2
        else:
            first = 'not_spotted'

        for report_id in commits_info_data[commit_id]['report_ids']:
            if report_id not in per_vulnerability:
                per_vulnerability[report_id]=[]

            matched_ref_obj = _get_matched_ref_object_by_commit_id(commit_id, first)
            per_vulnerability[report_id].append(matched_ref_obj)

    for x in sighted_vulnerabilities:
        data[x]['fb_commit_classification'] = per_vulnerability[x]

def load_vfm_commit_classification(data):
    global commits_info_data

    with open(vfm_classified_commits, 'r') as f:
        vfm_classified = json.load(f)

    sighted_vulnerabilities = set()
    per_vulnerability_msg = {}
    per_vulnerability_patch = {}
    per_vulnerability_combined = {}
    for x in vfm_classified:
        if not x['is_security_related']:
            continue

        if x['id'] not in commits_info_data:
            continue

        commit_id = x['id']
        commit = commits_info_data[x['id']]['commit_info']
        sighted_vulnerabilities.update(commits_info_data[x['id']]['report_ids'])

        date1 = parser.parse(commit['commit']['committer']['date'])
        date2 = parser.parse(commit['commit']['author']['date'])
        first = date1 if date1 < date2 else date2
        if x['msg_prob'][0] >= 0.5 :
            spotted_date = first
        else:
            spotted_date = 'not_spotted'
        for report_id in commits_info_data[x['id']]['report_ids']:
            if report_id not in per_vulnerability_msg:
                per_vulnerability_msg[report_id]=[]

            matched_ref_obj = _get_matched_ref_object_by_commit_id(commit_id, spotted_date)
            per_vulnerability_msg[report_id].append(matched_ref_obj)

        if x['patch_prob'][0] >= 0.5 :
            spotted_date = first
        else:
            spotted_date = 'not_spotted'
        for report_id in commits_info_data[x['id']]['report_ids']:
            if report_id not in per_vulnerability_patch:
                per_vulnerability_patch[report_id]=[]
            matched_ref_obj = _get_matched_ref_object_by_commit_id(commit_id, spotted_date)
            per_vulnerability_patch[report_id].append(matched_ref_obj)

        if x['msg_prob'][0]*x['msg_prob'][0] + x['patch_prob'][0]*x['patch_prob'][0] >= 0.25 :
            spotted_date = first
        else:
            spotted_date = 'not_spotted'
        for report_id in commits_info_data[x['id']]['report_ids']:
            if report_id not in per_vulnerability_combined:
                per_vulnerability_combined[report_id]=[]
            matched_ref_obj = _get_matched_ref_object_by_commit_id(commit_id, spotted_date)
            per_vulnerability_combined[report_id].append(matched_ref_obj)

    for x in sighted_vulnerabilities:
        data[x]['vfm_message_classification'] = per_vulnerability_msg[x]

    for x in sighted_vulnerabilities:
        data[x]['vfm_patch_classification'] = per_vulnerability_patch[x]

    for x in sighted_vulnerabilities:
        data[x]['vfm_combined_classification'] = per_vulnerability_combined[x]

def main():
    global commits_data
    # Import data on additional commits
    with open(r'src\recalls\data\commit_data_for_rq1.json', 'r') as f:
        commits_data = json.load(f)

    global commits_info_data
    # Import data on security commits
    with open(security_relevant_commits_info_file, 'r') as f:
        commits_info_data = json.load(f)

    # Import data from vulnerability databases
    nvd = NvdLoader(r'D:\Projects\VulnerabilityData\new_nvd')
    osv = OsvLoader(r'D:\Projects\VulnerabilityData\new_osv')
    ghsa = OsvLoader(r'D:\Projects\VulnerabilityData\advisory-database\advisories\github-reviewed')
    omni = OmniLoader(nvd, osv, ghsa)

    # Import data from fix mapper result
    references_data = {}
    data_files = get_files_in_from_directory(input_data_location, '.json')
    for data_file in data_files:
        with open(data_file, 'r') as f:
            data_temp = json.load(f)

        for report_id in data_temp:
            for x in data_temp[report_id]:
                x['data_obj'] = json.loads(x['data'])
                x['data'] = None
        for report_id in data_temp:
            references_data[report_id] = data_temp[report_id]

    data = {x:{} for x in omni.all_ids}

    # Import creation dates
    load_creation_date(data, omni)

    # Load ecosystems
    load_ecosystem(data, omni)

    # Load repositories
    load_repositories(data, references_data)

    # Import severity
    load_severity(data, omni)

    # Load all refs in new format
    load_all_refs(data, references_data)

    # Load first ref
    load_first_ref(data, references_data)

    # Load first issue
    load_first_issue(data, references_data)

    # Load first commit
    load_first_commit(data, references_data)

    # Load first ref with security phrases
    load_first_sec_phrase(data, references_data)

    # # Most found phrases
    # global phrases_to_ids
    # asd = sorted({k:len(v) for k,v in phrases_to_ids.items()}.items(), key=lambda x: x[1])

    # Load first issue with security label
    load_first_sec_label(data, references_data)

    # # Load first commit classified as security related (Feature-based)
    load_fb_commit_classification(data)

    # Load first commit classified as security related (own-models)
    # load_dl_commit_classification(data)

    # Load first commit classified as security related (VulFixMiner)
    load_vfm_commit_classification(data)

    with open('recall_analysis.json', 'w') as f:
        json.dump(data, f, default=str)

if __name__ == '__main__':
    main()
    print('ok')


#!/usr/bin/env python3
'''
Filename:
obtainMAData.py

Purpose:
This file is used to obtain and clean the data of new covid cases reported by
the state of Massachusetts. It will output the data into a file that can be
easily added to the database.

How to Execute:
$ python3 obtainMAData.py --output MAoutput.csv
'''

###################
# Import Libraries
###################

import sys
import argparse
import pandas as pd

############
# Main Code
############

def main():
    ''' Business Logic '''
    # Get command line arguments
    args = get_cli()
    print(f'The arguments are:\n{args}')

    # Make sure the output is acceptable
    if args.output == 'output.csv':
        print(f'{args.output} is already being used. Please try another filename',
              file=sys.stderr)
        sys.exit()

    # At some point here I want to add a web scraping function to download the
    # file from MA directly

    # Read in the worksheet
    ma_data = pd.read_excel(args.input, engine='openpyxl',
                            sheet_name='TestingByDate (Test Date)',
                            usecols='A, C:F, L, M, R, T')
    print(f'The data is:\n {ma_data.head()}')

def get_cli():
    '''
    Parameter: None
    Returns: cli (an instance of the command line arguments)
    Purpose: This function obtains the arguments passed into the command line
    '''
    parser = argparse.ArgumentParser(description = 'Clean MA Covid Data')

    # Add arguments
    parser.add_argument('-input', '--i', dest='input', type = str,
                        help = 'Name of file to download data into',
                        required = False,
                        default ='../data/covid-19-raw-data-12-27-2021.xlsx')

    parser.add_argument('-output', '--o', dest = 'output', type = str,
                        help = 'Name of Output File', required = False,
                        default = '../data/MAoutput.csv')

    return parser.parse_args()

if __name__ == '__main__':
    main()


import ssl
import socket
import time,datetime
import requests
from requests.exceptions import ReadTimeout
import json
import re
from json import JSONDecodeError
from config import *
from pyquery import PyQuery as pq

class Cert_check(object):
    def __init__(self,domain):
        self.domain = domain
        self.sslchaoshi = 'https://api-tools.sslchaoshi.com/ssltools/sslVerity'
        self.myssl = 'https://myssl.com/api/v1/ssl_status?domain=' + self.domain + '&port=443&c=0'
        self.data = {
            'endDate':None,
            'expire':None
        }

    def py_check(self):
        try:
            ctx = ssl.create_default_context()
            s = ctx.wrap_socket(socket.socket(), server_hostname=self.domain)
            s.settimeout(5)
            s.connect((self.domain, 443))
            cert = s.getpeercert()
            # 有效期结束
            notAfter = time.mktime(time.strptime(cert['notAfter'], '%b %d %H:%M:%S %Y %Z'))
            # 当前时间
            nowDate = time.time()
            # 时间差
            self.data['expire'] = int((notAfter - nowDate) / 3600 / 24)
            # 时间格式本地化
            self.data['endDate'] = time.strftime("%Y-%m-%d", time.localtime(notAfter))
            # print(self.domain,self.data)
            return self.data
        # except socket.gaierror as e:
        #     print(self.domain,e)
        # except socket.timeout as e:
        #     print(self.domain,e)
        # except ssl.SSLError as e:
        #     print(self.domain,e)
        # except ssl.CertificateError as e:
        #     print(self.domain,e)
        except Exception:
            chacuo_result = self.getChacuoPage()
            if chacuo_result:
                return chacuo_result

            myssl_result = self.getmyssl()
            if myssl_result:
                return myssl_result

            sslchaoshi_result = self.getsslchaoshi()
            if sslchaoshi_result:
                return sslchaoshi_result
            else:
                return self.data

    # 从 https://myssl.com/ 爬取证书信息
    def getmyssl(self):
        print("myssl正在解析：%s" % self.domain)

        # 构建请求url,并发送请求
        url = 'https://myssl.com/api/v1/ssl_status?domain=' + self.domain + '&port=443&c=0'
        try:
            response = requests.get(url, timeout=20)
        except ReadTimeout:
            return False

        # 挑选出证书信息的html
        try:
            CA_info_d = json.loads(response.text)
        except JSONDecodeError:
            return False

        # 如果CA_info_doc为空,证明获取证书信息失败
        if "data" in CA_info_d:
            try:
                valid_d = CA_info_d['data']["status"]["certs"]["rsas"][0]["leaf_cert_info"]
            except IndexError:
                valid_d = CA_info_d['data']["status"]["certs"]["eccs"][0]["leaf_cert_info"]
                return False
            except KeyError:
                return False

            endValid_s = valid_d["valid_to"].split("T")[0]
            endValid_l = endValid_s.split("-")
            endValid = datetime.datetime(int(endValid_l[0]), int(endValid_l[1]), int(endValid_l[2]))
            valid = endValid - datetime.datetime.today()
            self.data['endDate'] = endValid_s
            self.data['expire'] = valid.days
            return self.data
        else:
            return False

    def getsslchaoshi(self):
        print("sslchaoshi正在解析：%s" % self.domain)

        # 设置请求url和需要POST的数据,并发送POST请求,会返回一个字典
        url = "https://api-tools.sslchaoshi.com/ssltools/sslVerity"
        data = {'hostname': self.domain, "port": 443}
        try:
            response = requests.post(url, data, timeout=20)
        except ReadTimeout:
            return False

        ssl_data = json.loads(response.text)

        # 如果有cerInfo这个key,说明获取证书信息成功,
        if "cerInfo" in ssl_data:
            # 获取证书信息成功,构建正确证书信息字典,并存入mongoDB
            to = ssl_data["cerInfo"]['activeTimeStamp']["to"]
            restTime = ssl_data["cerInfo"]['activeTimeStamp']["restTime"]
            self.data['endDate'] = time.strftime("%Y-%m-%d", time.localtime(to))
            self.data['expire'] = int(re.sub(r"天", "", restTime))

            return self.data
        else:
            return False

    # 从 http://web.chacuo.net/netsslcheck 爬去证书信息
    def getChacuoPage(self):
        print("chacuo正在解析：%s" % self.domain)

        # 拼接url
        # domain是需要检测的域名
        url = "http://web.chacuo.net/netsslcheck?type=sslcheck&data=" + self.domain

        # 发送请求
        try:
            response = requests.get(url, timeout=20)
        except ReadTimeout:
            return False
        except ConnectionError:
            return False

        # 将返回内容转换为字典,如果出现JSONDecodeError就从新调用当前函数再请求一次
        try:
            data = json.loads(response.text)
        except JSONDecodeError:
            return False

        # 如果返回的字典里的data这个key没有值,说明获取证书信息失败,
        if len(data["data"][0]) == 0:
            return False
        domainInfo_list = []
        # 解析数据,将有值的数据追加到domainInfo_list中
        doc = pq(data['data'][0])
        for i in doc("tr").items():
            info = i.text().split("\n")
            if len(info) == 2:
                domainInfo_list.append(info)

        # 将domainInfo_list转换为字典,并插入domain字段,保存到mongodb中
        detail = dict(domainInfo_list)
        self.data['endDate'] = detail['有效时间：'].split("到")[1]
        self.data['expire'] = int(re.search(r"\d+", detail['过期时间：：']).group())
        return self.data

if __name__ == '__main__':
    response = requests.post("http://"+ SERVER +":"+ PORT +"/webmoni/api/domain_all/",data={'node':NODE})
    response_dict = json.loads(response.text)

    for domain_obj in response_dict['data']:

        obj = Cert_check(domain_obj['url'])
        data = obj.py_check()
        print(domain_obj['url'],data)


import logging
from calendar import day_abbr, month_abbr, month_name
from datetime import date, datetime, timedelta
from typing import List, Optional, Tuple, Dict

import discord
from discord import app_commands
import requests
from dateutil.rrule import rrulestr
from discord.ext import commands
from icalendar import Calendar
from pytz import timezone, utc

from uqcsbot.bot import UQCSBot

UQCS_CALENDAR_URL = (
    "https://calendar.google.com/calendar/ical/"
    "q3n3pce86072n9knt3pt65fhio%40group.calendar.google.com/public/basic.ics"
)
# EXTERNAL_CALENDAR_URL =d "https://calendar.google.com/calendar/ical/" \
#                         "72abf01afvsl3bjd9oq2g1avgg%40group.calendar.google.com/public/basic.ics"
# Testing calendar: "https://calendar.google.com/calendar/ical/7djv171v2mdr4dmufq612j6uj4%40group.calendar.google.com/public/basic.ics"

MONTH_NUMBER: Dict[str, int] = {
    month.lower(): index for index, month in enumerate(month_abbr)
}

BRISBANE_TZ = timezone("Australia/Brisbane")

MAX_RECURRING_EVENTS = 3

class EventFilter(object):
    def __init__(self, full=False, weeks=None, cap=None, month=None, is_valid=True):
        self.is_valid = is_valid
        self._full = full
        self._weeks = weeks
        self._cap = cap
        self._month = month

    @classmethod
    def from_argument(cls, args: Tuple):
        if len(args) == 0:
            return cls(weeks=2)
        else:
            if "full" in args or "all" in args:
                return cls(full=True)
            elif "weeks" in args:
                i = args.index("weeks") + 1
                return cls(weeks=int(args[i]))
            else:
                # No valid input
                return cls(weeks=2)

    def filter_events(self, events: List["Event"], start_time: datetime):
        if self._weeks is not None:
            end_time = start_time + timedelta(weeks=self._weeks)
            return [e for e in events if e.start < end_time]
        if self._month is not None:
            return [e for e in events if e.start.month == self._month]
        elif self._cap is not None:
            return events[: self._cap]
        return events

    def get_header(self):
        if self._full:
            return "List of *all* upcoming events:"
        elif self._weeks is not None:
            return f"Events in the next *{self._weeks} weeks*:"
        elif self._month is not None:
            return f"Events in *{month_name[self._month]}*:"
        else:
            return f"The *next {self._cap} events*:"

    def get_no_result_msg(self):
        if self._weeks is not None:
            return (
                f"There don't appear to be any events in the next *{self._weeks}* weeks"
            )
        elif self._month is not None:
            return f"There don't appear to be any events in *{month_name[self._month]}*"
        else:
            return "There don't appear to be any upcoming events..."

class Event(object):
    def __init__(
        self,
        start: datetime,
        end: datetime,
        location: str,
        summary: str,
        recurring: bool,
        link: Optional[str],
        source: Optional[str] = None,
    ):
        self.start = start
        self.end = end
        self.location = location
        self.summary = summary
        self.recurring = recurring
        self.link = link
        self.source = source

    @classmethod
    def encode_text(cls, text: str) -> str:
        """
        Encodes user-specified text so that it is not interpreted as command characters
        by Slack. Implementation as required by: https://api.slack.com/docs/message-formatting
        Note that this encoding process does not stop injection of text effects (bolding,
        underlining, etc.), or a malicious user breaking the text formatting in the events
        command. It should, however, prevent <, & and > being misinterpreted and including
        links where they should not.
        --
        :param text: The text to encode
        :return: The encoded text
        """
        return text.replace("&", "&").replace("<", "<").replace(">", ">")

    @classmethod
    def from_cal_event(
        cls, cal_event, source: str = "UQCS", recurrence_dt: datetime = None
    ):
        """
        Converts an ical event to an Event

        :param cal_event: event to convert
        :param source: the calendar the event was sourced from
        :param recurrence_dt: if this is a one off event then None
                                else the date of this instance of a recurring event
        """
        if recurrence_dt:
            start = recurrence_dt
            end = recurrence_dt + (
                cal_event.get("DTEND").dt - cal_event.get("DTSTART").dt
            )

        else:
            start = cal_event.get("dtstart").dt
            end = cal_event.get("dtend").dt
            # ical 'dt' properties are parsed as a 'DDD' (datetime, date, duration) type.
            # The below code converts a date to a datetime, where time is set to midnight.
            if isinstance(start, date) and not isinstance(start, datetime):
                start = datetime.combine(start, datetime.min.time()).astimezone(utc)
            if isinstance(end, date) and not isinstance(end, datetime):
                end = datetime.combine(end, datetime.max.time()).astimezone(utc)
        location = cal_event.get("location", "TBA")
        summary = cal_event.get("summary")
        return cls(
            start,
            end,
            location,
            f"{'[External] ' if source == 'external' else ''}{summary}",
            recurrence_dt is not None,
            None,
            source,
        )

    def get_title(self):
        # TODO: fix for case with link
        return Event.encode_text(
            ("[Recurring] " if self.recurring else "") + self.summary
        )

    def get_time_loc(self):
        d1 = self.start.astimezone(BRISBANE_TZ)
        d2 = self.end.astimezone(BRISBANE_TZ)

        start_str = (
            f"{day_abbr[d1.weekday()].upper()}"
            + f" {month_abbr[d1.month].upper()} {d1.day} {d1.hour}:{d1.minute:02}"
        )
        if (d1.month, d1.day) != (d2.month, d2.day):
            end_str = (
                f"{day_abbr[d2.weekday()].upper()}"
                + f" {month_abbr[d2.month].upper()} {d2.day} {d2.hour}:{d2.minute:02}"
            )
        else:
            end_str = f"{d2.hour}:{d2.minute:02}"

        # Encode user-provided text to prevent certain characters
        # being interpreted as slack commands.
        location_str = Event.encode_text(self.location)
        return f"**{start_str} - {end_str}** {'*(' + location_str + ')*' if location_str else ''}"

class Events(commands.Cog):
    """
    Display events
    """

    CHANNEL_NAME = "events"

    def __init__(self, bot: UQCSBot):
        self.bot = bot
        self.bot.schedule_task(
            self.scheduled_message,
            trigger="cron",
            hour=9,
            day_of_week="mon",
            timezone="Australia/Brisbane",
        )

    async def scheduled_message(self):
        channel = discord.utils.get(
            self.bot.uqcs_server.channels, name=self.CHANNEL_NAME
        )
        if channel is not None:
            await self.send_events(channel)
        else:
            logging.warning(f"Could not find required channel #{self.CHANNEL_NAME}")

    @classmethod
    def _get_current_time(cls):
        """
        Returns the current date and time
        This function exists purely so it can be mocked for testing
        """
        return datetime.now(tz=BRISBANE_TZ).astimezone(utc)

    def _handle_calendar(self, calendar) -> List[Event]:
        """
        Returns a list of events from a calendar
        """
        events = []
        current_time = self._get_current_time()
        # subcomponents are how icalendar returns the list of things in the calendar
        for component in calendar.subcomponents:
            # we are only interested in ones with the name VEVENT as they
            # are events
            if component.name != "VEVENT":
                continue
            elif component.get("RRULE") is not None:
                # If the until date exists, update it to UTC
                if component["RRULE"].get("UNTIL") is not None:
                    until = datetime.combine(
                        component["RRULE"]["UNTIL"][0], datetime.min.time()
                    ).astimezone(utc)
                    component["RRULE"]["UNTIL"] = [until]
                rule = rrulestr(
                    "\n".join(
                        [
                            line
                            for line in component.content_lines()
                            if line.startswith("RRULE") or line.startswith("EXDATE")
                        ]
                    ),
                    dtstart=component.get("DTSTART").dt,
                )
                rule = [dt for dt in list(rule) if dt > current_time]
                for dt in rule[:MAX_RECURRING_EVENTS]:
                    dt = dt.replace(tzinfo=BRISBANE_TZ)
                    event = Event.from_cal_event(component, recurrence_dt=dt)
                    events.append(event)
            else:
                # we convert it to our own event class
                event = Event.from_cal_event(component)
                # then we want to filter out any events that are not after the current time
                if event.start > current_time:
                    events.append(event)

        return events

    async def send_events(
        self,
        channel: discord.abc.Messageable,
        interaction: discord.Interaction = None,
        *args,
    ):
        current_time = self._get_current_time()
        event_filter = EventFilter.from_argument(args)
        events = []

        uqcs_calendar = Calendar.from_ical(self._get_calendar_file())
        events += self._handle_calendar(uqcs_calendar)

        # then we apply our event filter as generated earlier
        events = event_filter.filter_events(events, current_time)
        # then, we sort the events by date
        events = sorted(events, key=lambda event_: event_.start)

        if not events:
            message_text = (
                f"_{event_filter.get_no_result_msg()}_\n"
                f"For a full list of events, visit: "
                f"https://uqcs.org/events "
            )
            if interaction == None:
                await channel.send(message_text)
            else:
                await interaction.response.send_message(message_text)
        else:
            message_text = f"{event_filter.get_header()}"
            if interaction == None:
                await channel.send(message_text)
            else:
                await interaction.response.send_message(message_text)

            for event in events:
                colour = discord.Colour.from_rgb(82, 151, 209)

                embed = discord.Embed()
                embed.colour = colour
                embed.title = f"{event.get_title()}"
                embed.description = f"{event.get_time_loc()}"
                await channel.send(embed=embed)

    @app_commands.command()
    @app_commands.describe(weeks="Number of weeks to return, defaults to 2")
    async def events(self, interaction: discord.Interaction, weeks: int = 2):
        """Shows the upcoming UQCS events for the next 2 weeks."""
        await self.send_events(interaction.channel, interaction, "weeks", weeks)

    @app_commands.command()
    async def allevents(self, interaction: discord.Interaction):
        """Shows all upcoming UQCS events."""
        await self.send_events(interaction.channel, interaction, "full")

    @classmethod
    def _get_calendar_file(cls) -> bytes:
        """
        Loads the UQCS or External Events calender .ics file from Google Calendar.
        This method is mocked by unit tests.
        :return: The returned ics calendar file, as a stream
        """
        http_response = requests.get(UQCS_CALENDAR_URL)
        return http_response.content

async def setup(bot: commands.Bot):
    await bot.add_cog(Events(bot))


from matplotlib.figure import Figure
from matplotlib.pyplot import imread
from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas
from numpy import ndarray
from PyQt5.QtCore import QThread, pyqtSignal

class ImgFigure(FigureCanvas):
    def __init__(self, dpi=100):
        self.fig = Figure(dpi=dpi)
        super(ImgFigure, self).__init__(self.fig)
        self.axes = self.fig.add_subplot(111)
        self.axes.get_yaxis().set_visible(False)
        self.axes.get_xaxis().set_visible(False)

class WorkThread(QThread):

    finishSignal = pyqtSignal(ndarray, ndarray)

    def __init__(self, imgPath, progressBar, alpha, gamma):
        super(WorkThread, self).__init__()

        from LIME import LIME
        img = imread(imgPath)
        self.lime = LIME(img, alpha, gamma)
        self.lime.setMaximumSignal.connect(progressBar.setMaximum)
        self.lime.setValueSignal.connect(progressBar.setValue)

    def run(self):
        T = self.lime.optimizeIllumMap()
        R = self.lime.enhance()
        self.finishSignal.emit(T, R)


from django.shortcuts import render
from stove.models import State
from django.http import HttpResponse, JsonResponse
from django.views.decorators.csrf import csrf_exempt

import json
# Create your views here.

@csrf_exempt
def index(request):
    return render(request, 'index.html')

def getState(request):
    result =  State.get_state()
    #print("from result", result["temp"])

    if (result['temp'] > 30 and result['proximity1'] > 15):
        display_msg= "Stove seems to be ON and nothing is cooking!"
    elif (result['temp'] > 30 and result['proximity1'] < 15):
        display_msg = "Food on stove. Must be burning!"
    else:
        display_msg = "Everything is OK"

    context = {
    'temp' : result['temp'],
    'proximity1' : result['proximity1'],
    'proximity2' : result['proximity2'],
    'display_msg' : display_msg,
        }

    #request.session['temp'] = result.temp
    #request.session['proximity1'] = result.proximity1
    #request.session['proximity2'] = result.proximity2
    return render(request,'dashboard.html', context=context)
    #return JsonResponse(State.get_state())

def mobileApp(request):
    return JsonResponse(State.get_state())

def saveState(request, temp, p1, p2):
    # temp = float(request.POST.get('temp'))
    # p1 = float(request.POST.get('p1'))
    # p2 = float(request.POST.get('p2'))
    created = State.saveState(float(temp), float(p1), float(p2))
    return JsonResponse({"success": created})

Web Server and Client/smart_stove/stove/views.py


#!/usr/bin/env python

import numpy as np
from sklearn.model_selection import KFold
from scipy import interpolate

def sorted_nparray_last_argmin(arr, min_val=None):
    if min_val is None:
        min_val = arr.min()

    for idx in range(len(arr)):
        if arr[idx] > min_val:
            min_idx = idx - 1
            break

    return min_idx

def sorted_nparray_first_argmax(arr, max_val=None):
    if max_val is None:
        max_val = arr.max()

    for idx in np.arange(len(arr) - 1, -1, -1):
        if arr[idx] < max_val:
            max_idx = idx + 1
            break

    return max_idx

def calc_accuracy(threshold, dist, actual_issame):
    """
        Calculate accuracy at some threshold
    """

    predict_issame = np.less(dist, threshold)

    tp = np.sum(np.logical_and(predict_issame, actual_issame))
    fp = np.sum(np.logical_and(predict_issame, np.logical_not(actual_issame)))

    tn = np.sum(np.logical_and(np.logical_not(
        predict_issame), np.logical_not(actual_issame)))
    fn = np.sum(np.logical_and(np.logical_not(predict_issame), actual_issame))

    tpr = 0 if (tp + fn == 0) else float(tp) / float(tp + fn)
    fpr = 0 if (fp + tn == 0) else float(fp) / float(fp + tn)
    acc = float(tp + tn) / dist.size

    return tpr, fpr, acc

def calc_val_far(threshold, dist, actual_issame):
    """
        Calculate VAL(=TPR) at some threshold
    """

    predict_issame = np.less(dist, threshold)

    true_accept = np.sum(np.logical_and(predict_issame, actual_issame))

    false_accept = np.sum(np.logical_and(
        predict_issame, np.logical_not(actual_issame)))

    n_same = np.sum(actual_issame)
    n_diff = np.sum(np.logical_not(actual_issame))

    val = float(true_accept) / float(n_same)
    far = float(false_accept) / float(n_diff)

    return val, far

def calc_roc(thresholds,
             dist,
             actual_issame):
    """
        ROC evaluation without k-folds
    """

    if thresholds is None or thresholds == []:
        thresholds = np.sort(dist)

    nrof_thresholds = len(thresholds)

    tprs = np.zeros(nrof_thresholds)
    fprs = np.zeros(nrof_thresholds)

    accuracies = np.zeros(nrof_thresholds)

    # Find the best threshold
    for idx, threshold in enumerate(thresholds):
        tprs[idx], fprs[idx], accuracies[idx] = calc_accuracy(
            threshold, dist, actual_issame)

    best_thresh_idx = np.argmax(accuracies)
    best_thresh = thresholds[best_thresh_idx]
    accuracy = accuracies[best_thresh_idx]

#    print("\t best_thresh_idx = %d" % (best_thresh_idx))
#    print("\t best_threshold = %2.5f" % (thresholds[best_thresh_idx]))
#    print("\t accuracy = %2.5f" % (accuracy))
#

    val, far = calc_val_far(best_thresh, dist, actual_issame)

    best_accuracy = {
        "accuracy": accuracy,
        "threshold": best_thresh,
        "VAL": val,
        "FAR": far
    }

    return tprs, fprs, best_accuracy

def calc_val(thresholds,
             dist,
             actual_issame,
             far_targets=None):
    """
        Calculate VAL(=TPR) at target FARs without k-folds
    """

    if thresholds is None or thresholds == []:
        thresholds = np.sort(dist)

    if not far_targets:
        far_targets = [0.1, 0.01, 0.001, 0.0001, 0.00001, 0]

    nrof_thresholds = len(thresholds)

    val_array = np.zeros(nrof_thresholds)
    far_array = np.zeros(nrof_thresholds)

    # Find the threshold that gives FAR = far_target
    for idx, threshold in enumerate(thresholds):
        val_array[idx], far_array[idx] = calc_val_far(
            threshold, dist, actual_issame)

#    min_far_idx = np.argmin(far_array)
#    max_far_idx = np.argmax(far_array)
#    min_far = far_array[min_far_idx]
#    max_far = far_array[max_far_idx]
#    min_far = np.min(far_array)
#    max_far = np.max(far_array)

    # find the last idx with min_far
#    for idx in range(len(far_array)):
#        if far_array[idx] > min_far:
#            min_far_idx = idx - 1
#            break
    min_far_idx = sorted_nparray_last_argmin(far_array)

    # find the first idx with max_far
#    for idx in np.arange(len(far_array) - 1, -1, -1):
#        if far_array[idx] < max_far:
#            max_far_idx = idx + 1
#            break
    max_far_idx = sorted_nparray_first_argmax(far_array)
    min_far = far_array[min_far_idx]
    max_far = far_array[max_far_idx]

#    print('min_far=%2.5f, max_far=%2.5f' % (min_far, max_far))
#    print('min_far_idx=%d, max_far_idx=%d' % (min_far_idx, max_far_idx))

    f1 = interpolate.interp1d(far_array[min_far_idx:max_far_idx],
                              val_array[min_far_idx:max_far_idx])
    f2 = interpolate.interp1d(far_array[min_far_idx:max_far_idx],
                              thresholds[min_far_idx:max_far_idx],
                              kind='slinear')

    outputs = []

    for idx, far_t in enumerate(far_targets):
        #        print('Calc VAL and threshold @ FAR=%2.5f' % far_t)
        if far_t >= max_far:
            val = val_array[max_far_idx]
            thresh = thresholds[max_far_idx]
        elif far_t <= min_far:
            val = val_array[min_far_idx]
            thresh = thresholds[min_far_idx]
        else:
            val = f1(far_t)
            thresh = f2(far_t)

        t_dict = {
            'FAR': far_t,
            'VAL': val.tolist(),
            'threshold': thresh.tolist()
        }

#        print t_dict

        outputs.append(t_dict)

    return outputs

def calc_roc_kfolds(thresholds,
                    dist,
                    actual_issame,
                    distance='cosine',
                    nrof_folds=10):
    """
        ROC evaluation with k-folds
    """

    if thresholds is None or thresholds == []:
        thresholds = np.sort(dist)

    nrof_thresholds = len(thresholds)

    tprs = np.zeros((nrof_folds, nrof_thresholds))
    fprs = np.zeros((nrof_folds, nrof_thresholds))

    nrof_pairs = min(len(actual_issame), dist.shape[0])

    k_fold = KFold(n_splits=nrof_folds, shuffle=False)

    accuracy = np.zeros((nrof_folds))

    indices = np.arange(nrof_pairs)

    for fold_idx, (train_set, test_set) in enumerate(k_fold.split(indices)):
        #        print("===>Kfold: fold_idx=%d" % fold_idx)

        # Find the best threshold for the fold
        acc_train = np.zeros((nrof_thresholds))
        for idx, threshold in enumerate(thresholds):
            _, _, acc_train[idx] = calc_accuracy(
                threshold, dist[train_set], actual_issame[train_set])

        best_thresh_idx = np.argmax(acc_train)

#        print("\t best_thresh_idx = %d" % (best_thresh_idx))
#        print("\t best_threshold = %2.5f" % (thresholds[best_thresh_idx]))

        for idx, threshold in enumerate(thresholds):
            tprs[fold_idx, idx], fprs[fold_idx, idx], _ = calc_accuracy(
                threshold, dist[test_set], actual_issame[test_set])

        _, _, accuracy[fold_idx] = calc_accuracy(
            thresholds[best_thresh_idx], dist[test_set], actual_issame[test_set])

#        print("\t accuracy = %2.5f" % (accuracy[fold_idx]))

        tpr = np.mean(tprs, 0)
        fpr = np.mean(fprs, 0)

    return tpr, fpr, accuracy

def calc_val_kfolds(thresholds,
                    dist,
                    actual_issame,
                    far_target,
                    distance='cosine',
                    nrof_folds=10):
    """
        Calculate VAL(=TPR) at target FARs with k-folds
    """

    if thresholds is None or thresholds == []:
        thresholds = np.sort(dist)

    nrof_thresholds = len(thresholds)

    nrof_pairs = min(len(actual_issame), dist.shape[0])
    k_fold = KFold(n_splits=nrof_folds, shuffle=False)

    val = np.zeros(nrof_folds)
    far = np.zeros(nrof_folds)

    indices = np.arange(nrof_pairs)

    for fold_idx, (train_set, test_set) in enumerate(k_fold.split(indices)):
        far_train = np.zeros(nrof_thresholds)

        for idx, threshold in enumerate(thresholds):
            _, far_train[idx] = calc_val_far(
                threshold, dist[train_set], actual_issame[train_set])

        min_far_idx = sorted_nparray_last_argmin(far_train)
        max_far_idx = sorted_nparray_first_argmax(far_train)
        min_far = far_train[min_far_idx]
        max_far = far_train[max_far_idx]

        if far_target >= max_far:
            threshold = 0.0
        elif far_target <= min_far:
            threshold = thresholds[min_far_idx]
        else:
            f = interpolate.interp1d(far_train[min_far_idx:max_far_idx],
                                     thresholds[min_far_idx:max_far_idx],
                                     kind='slinear')
            threshold = f(far_target)

        val[fold_idx], far[fold_idx] = calc_val_far(
            threshold, dist[test_set], actual_issame[test_set])

    val_mean = np.mean(val)
    far_mean = np.mean(far)
    val_std = np.std(val)

    return val_mean, val_std, far_mean


from itertools import product
import time
import re
import requests
from bs4 import BeautifulSoup
from .common import Common
from domains.models import Domain, Trademark, Url
from conditions.models import NG_Keyword_Condition
from products.models import Product
from results.models import Check_Result

class NgCheck(Common):
    def update_check_result_table(self, url, priority):
        try:
            result = Check_Result.objects.get(url=url)
            result.priority = priority
            result.confirmed = False
            result.save()
        except:
            Check_Result.objects.create(url=url, priority=priority, confirmed=False)

    def ng_check(self, middle_kws, target_urls):
        for target_url in target_urls:
            # self.driver.get(target_url.url)
            # time.sleep(1)
            # html_source = self.driver.page_source
            # text_source = BeautifulSoup(html_source, "html.parser").text
            try:
                res = requests.get(target_url.url)
                text_source = res.text
            except:
                print(f"サイトのテキストを取得できませんでした：{target_url.url}")
                continue
            if text_source == "":
                print(f"サイトのテキストを取得できませんでした：{target_url.url}")
                continue
            for middle_kw in middle_kws:
                priority = 3
                ng_kw = str(middle_kw.ng_keyword)
                composite_kw = str(middle_kw.composite_keyword)
                front_word_count = middle_kw.front_check_word_count
                back_word_count = middle_kw.back_check_word_count
                ng_kw_check_list = list(re.finditer(ng_kw, text_source))
                if len(ng_kw_check_list) > 0 and composite_kw is not None:
                    priority = 2
                    for ng_kw in ng_kw_check_list:
                        ng_kw_start_index = ng_kw.start()
                        ng_kw_start_index = ng_kw.start()
                        ng_kw_end_index = ng_kw.end()
                        front_start_index = max(0, ng_kw_start_index - front_word_count - 1)
                        front_end_index = max(0, ng_kw_start_index - 1)
                        back_start_index = min(len(text_source), ng_kw_end_index + 1)
                        back_end_index = min(len(text_source), ng_kw_end_index + back_word_count + 1)
                        front_result = text_source.find(composite_kw, front_start_index, front_end_index)
                        back_result = text_source.find(composite_kw, back_start_index, back_end_index)
                        if  front_result != -1 and back_result != -1:
                            priority = 1
                            break
                self.update_check_result_table(url=target_url, priority=priority)

    def update_checkresult_status_to_judgement(self, domain):
        target_urls = Url.objects.filter(domain=domain)
        for target_url in target_urls:
            self.update_check_result_table(url=target_url, priority=4)

    def job(self, product_id):
        product_instance = Product.objects.get(id=product_id)
        trademark_kws = Trademark.objects.filter(product=product_instance)
        domains = []
        for trademark_kw in trademark_kws:
            domains += Domain.objects.filter(trademark=trademark_kw, _type=2)
        # domainsは、{domain:, trademark_kw:}の辞書型リスト
        for domain in domains:
            self.update_checkresult_status_to_judgement(domain=domain)
        print("success to update all urls flag to judgement !")
        for domain in domains:
            print(f"checking this domain: {domain}")
            client_target_urls = Url.objects.filter(domain=domain)
            middle_kws = NG_Keyword_Condition.objects.filter(product_condition=Product.objects.get(id=product_id).product_condition).exclude(status=0)
            self.ng_check(middle_kws=middle_kws, target_urls=client_target_urls)


import json

def input_number():
    """用户输入喜欢的数字"""
    try:
        number = int(input('Please in put your favorite number: '))
    except ValueError:
        print('Please input the number')
    else:
        filename = 'number_1.json'
        with open(filename, 'w') as file_object:
            json.dump(number,file_object)
        return number

def print_number():
    """打印用户喜欢的数字"""
    try:
        filename = 'number_1.json'
        with open(filename) as file_object:
            number = json.load(file_object)
    except FileNotFoundError:
        number = input_number()
        print('Your favorite is ' + str(number) + '!')
    else:
        print("I know your favorite number! It's " + str(number) + '!')

print_number()


# -*- coding:utf-8 -*-
from mako import runtime, filters, cache
UNDEFINED = runtime.UNDEFINED
__M_dict_builtin = dict
__M_locals_builtin = locals
_magic_number = 10
_modified_time = 1438909689.735831
_enable_loop = True
_template_filename = '/Users/mq20155490/Desktop/CEL/abstrackr_test/abstrackr/templates/reviews/show_review.mako'
_template_uri = '/reviews/show_review.mako'
_source_encoding = 'utf-8'
from webhelpers.html import escape
_exports = ['title']

def _mako_get_namespace(context, name):
    try:
        return context.namespaces[(__name__, name)]
    except KeyError:
        _mako_generate_namespaces(context)
        return context.namespaces[(__name__, name)]
def _mako_generate_namespaces(context):
    pass
def _mako_inherit(template, context):
    _mako_generate_namespaces(context)
    return runtime._inherit_from(context, u'../site.mako', _template_uri)
def render_body(context,**pageargs):
    __M_caller = context.caller_stack._push_frame()
    try:
        __M_locals = __M_dict_builtin(pageargs=pageargs)
        url = context.get('url', UNDEFINED)
        c = context.get('c', UNDEFINED)
        float = context.get('float', UNDEFINED)
        len = context.get('len', UNDEFINED)
        __M_writer = context.writer()
        __M_writer(u'\n')
        __M_writer(u'\n')
        __M_writer(u'\n\n\n\n
')
        __M_writer(escape(c.review.name))
        __M_writer(u'
\n\t
\n    review labels\n    review terms\n
\n
\n
Project description
 \n')
        __M_writer(escape(c.review.description))
        __M_writer(u'\n

\n
Progress
\n\n')
        if float(c.num_labels)/float(c.num_citations) >= .1:
            __M_writer(u'\t
\n')
        __M_writer(u'\nThere are ')
        __M_writer(escape(c.num_citations))
        __M_writer(u' citations in this review, so far ')
        __M_writer(escape(c.num_labels))
        __M_writer(u' have been labeled.\n

\n\n
Participants
\nThis review is lead by\n
\n
\n')
        for leader in c.project_leaders:
            __M_writer(u'
\n                ')
            __M_writer(escape(leader.fullname))
            __M_writer(u'\n
\n')
        __M_writer(u'
\n
\n
\n')
        if len(c.participating_reviewers) > 1:
            __M_writer(u'    The following people are reviewers on the project: \n')
            for user in c.participating_reviewers[:-1]:
                __M_writer(u'        ')
                __M_writer(escape(user.fullname))
                __M_writer(u',\n')
            __M_writer(u'    and ')
            __M_writer(escape(c.participating_reviewers[-1].fullname))
            __M_writer(u'.\n')
        else:
            __M_writer(u'    This project is a lonely undertaking by ')
            __M_writer(escape(c.participating_reviewers[0].fullname))
            __M_writer(u'.\n')
        __M_writer(u'

\nNumber of citations screened by reviewers:\n\n\n\n
\n')
        return ''
    finally:
        context.caller_stack._pop_frame()

def render_title(context):
    __M_caller = context.caller_stack._push_frame()
    try:
        c = context.get('c', UNDEFINED)
        __M_writer = context.writer()
        __M_writer(escape(c.review.name))
        return ''
    finally:
        context.caller_stack._pop_frame()

"""
__M_BEGIN_METADATA
{"source_encoding": "utf-8", "line_map": {"27": 0, "36": 1, "37": 2, "38": 3, "39": 7, "40": 7, "41": 10, "42": 10, "43": 12, "44": 12, "45": 16, "46": 16, "47": 20, "48": 21, "49": 21, "50": 21, "51": 23, "52": 24, "53": 24, "54": 24, "55": 24, "56": 31, "57": 32, "58": 33, "59": 33, "60": 36, "61": 39, "62": 40, "63": 41, "64": 42, "65": 42, "66": 42, "67": 44, "68": 44, "69": 44, "70": 45, "71": 46, "72": 46, "73": 46, "74": 48, "75": 51, "76": 51, "82": 3, "87": 3, "93": 87}, "uri": "/reviews/show_review.mako", "filename": "/Users/mq20155490/Desktop/CEL/abstrackr_test/abstrackr/templates/reviews/show_review.mako"}
__M_END_METADATA
"""


import json

from django.http import JsonResponse
from django.views import View

from owners.models import Owner, Dog

class OwnersView(View):
# """
# 목적: Client가 보내주는 owner정보(name, age, email) 등록 요청에 맞게 owner테이블 in database 에 저장

# Input:
#     {
# 'name'  = '홍길동'
# 'age'   = 20
# 'email' = asd@gmail.com
#     }

# Logic:
#     1. owner 모델을 사용해서 owner 테이블에 데이터 입력
#     2. Owner.objects.create(...)

# Output:
#     {
# "message": "SUCCESS"
#     }, status = 201

# """
    def post(self, request):
        data = json.loads(request.body)

        Owner.objects.create(
            name  = data['name'],
            age   = data['age'],
            email = data['email']
        )
        return JsonResponse({"message" : "SUCCESS"}, status = 201)

# """
# 목적: owners의 정보를 client 요청하는 바와 맞게 database에서 호출하고, 가공해서 응답(반환)

# Logic:
# 1. 응답해야할 방식에 맞춰주기 위해서 빈 리스트 생성 / results = []

# 2. 모든 owner의 정보 추출 / owners = Owner.Objects.all()

#     3. for loop으로 강아지 리스트(name , age 포함되도록)가 함께 응답될 수 있도록 짜야함

# Output:
#     {
# "results" = result
# },         status = 200

# """

# list comprehension
# dict comprehension

    def get(self, request):

        # 신버전 ver3
        results = [
            {
                    'name' : owner.name,
                    'age'  : owner.age,
                    'email': owner.email,
                    'dogs' : [{
                        'name': dog.name,
                        'age' : dog.age,
                    } for dog in owner.dog_set.all()]
            } for owner in Owner.objects.all()
        ]
        return JsonResponse({'results' : results}, status = 200)

        # 구버전 ver 2
        results = []
        owners  = Owner.objects.all()

        for owner in owners:
            results.append(
                {
                    'name' : owner.name,
                    'age'  : owner.age,
                    'email': owner.email,
                    'dogs' : [{
                        'name': dog.name,
                        'age' : dog.age,
                    } for dog in owner.dog_set.all()]
                }
            )

        # 구버전 ver 1
        for owner in owners:
            dogs = list(Dog.objects.filter(owner=owner.id).values('name', 'age'))
            results.append(
                {
                    'name' : owner.name,
                    'age'  : owner.age,
                    'email': owner.email,
                    'dogs' : dogs,
                }
            )

class DogsView(View):

# """
# 목적: Client가 보내주는 dog정보(name, age, owner_id) 등록 요청에 맞게 dog테이블 in database 에 저장

# Input:
#     {
# 'name'  = '라온'
# 'age'   = 1
# 'owner' = '홍길동'
#     }

# Logic:
#     1. dog 모델을 사용해서 dog 테이블에 데이터 입력
#     2. Dog.objects.create()

# Output:
#     {
# "message": "SUCCESS"
#     }, status = 201

# """

    def post(self, request):
        data = json.loads(request.body)

        owner = Owner.objects.get(name=data['owner'])

        Dog.objects.create(
            name     = data['name'],
            age      = data['age'],
            owner_id = owner.id
        )
        return JsonResponse({"message" : "SUCCESS"}, status = 201)

# """
# 목적: dogs의 정보를 client 요청하는 바와 맞게 database에서 호출하고, 가공해서 응답(반환)

# Logic:
# 1. 응답해야할 방식에 맞춰주기 위해서 빈 리스트 생성 / results = []

# 2. 모든 dog의 정보 추출 / dogs = Dog.Objects.all()

# Output:
#     {
# "results" = result
# },         status = 200

# """

    def get(self, request):
        results = []
        dogs    = Dog.objects.all()

        for dog in dogs:
            results.append(
                {
                    'name' : dog.name,
                    'age'  : dog.age,
                    'owner': dog.owner.name
                }
            )
        return JsonResponse({"results" : results}, status=200)


from collections import defaultdict

import matplotlib.pyplot as plt
import numpy as np

from hrv.classical import time_domain
from hrv.rri import RRi

__all__ = ["time_varying"]

class TimeVarying:
    def __init__(self, rri, results, rri_segments, seg_size, overlap):
        self.rri = rri
        self.results = results
        self.transponsed = self._transform_results(results)
        self.rri_segments = rri_segments
        self.seg_size = seg_size
        self.overlap = overlap

    def _transform_results(self, results):
        return {key: [item[key] for item in results] for key in results[0].keys()}

    def ylabel_mapper(self, index):
        mapper = {
            "rmssd": "RMSSD (ms)",
            "sdnn": "SDNN (ms²)",
            "sdsd": "SDSD (ms²)",
            "nn50": "nn50 (count)",
            "pnn50": "pnn50 (%)",
            "mrri": "mean RRi (ms)",
            "mhr": "mean HR (bpm)",
        }
        return mapper.get(index)

    def build_xaxis(self):
        return [np.median(r.time) for r in self.rri_segments]

    def __getattr__(self, index):
        try:
            return self.transponsed[index]
        except KeyError:
            raise ValueError(f"index `{index}` does not exist.")

    def plot(self, ax=None, index="rmssd", *args, **kwargs):
        fig = None
        if ax is None:
            fig, ax = plt.subplots(1, 1)

        ax.plot(self.build_xaxis(), self.__getattr__(index), *args, **kwargs)
        ax.set(xlabel="Time Interval (s)", ylabel=self.ylabel_mapper(index))
        plt.show(block=False)

        return fig, ax

    def plot_together(self, ax=None, index="rmssd", *args, **kwargs):
        fig, ax = self.rri.plot(alpha=0.7, color="grey")
        ax1 = ax.twinx()
        self.plot(ax=ax1, index=index, *args, **kwargs)

        return fig, ax

    def __str__(self):
        return f"Time Varying {self.seg_size}:{self.overlap} - #{len(self.results)}"

def time_varying(rri, seg_size, overlap, keep_last=False):
    if not isinstance(rri, RRi):
        rri = RRi(rri)

    segments = rri.time_split(seg_size=seg_size, overlap=overlap, keep_last=keep_last)
    results = []
    for segment in segments:
        results.append(time_domain(segment))

    return TimeVarying(rri, results, segments, seg_size=seg_size, overlap=overlap)