import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve, precision_recall_curve, average_precision_score
from sklearn.metrics import auc
import numpy as np

def caculate_metric(pred_prob, label_pred, label_real):
    test_num = len(label_real)
    tp = 0
    fp = 0
    tn = 0
    fn = 0

    for index in range(test_num):
        if label_real[index] == 1:
            if label_real[index] == label_pred[index]:
                tp = tp + 1
            else:
                fn = fn + 1
        else:
            if label_real[index] == label_pred[index]:
                tn = tn + 1
            else:
                fp = fp + 1

    # Accuracy
    ACC = float(tp + tn) / test_num

    # Sensitivity
    if tp + fn == 0:
        Recall = Sensitivity = 0
    else:
        Recall = Sensitivity = float(tp) / (tp + fn)

    # Specificity
    if tn + fp == 0:
        Specificity = 0
    else:
        Specificity = float(tn) / (tn + fp)

    # MCC
    if (tp + fp) * (tp + fn) * (tn + fp) * (tn + fn) == 0:
        MCC = 0
    else:
        MCC = float(tp * tn - fp * fn) / (np.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn)))

    # ROC and AUC
    FPR, TPR, thresholds = roc_curve(label_real, pred_prob, pos_label=1)  # Default 1 is positive sample
    AUC = auc(FPR, TPR)

    # PRC and AP
    # precision, recall, thresholds = precision_recall_curve(label_real, pred_prob, pos_label=1)
    # AP = average_precision_score(label_real, pred_prob, average='macro', pos_label=1, sample_weight=None)

    # ROC(FPR, TPR, AUC)
    # PRC(Recall, Precision, AP)

    performance = [ACC, Sensitivity, Specificity, AUC, MCC]
    roc_data = None
    prc_data = None
    # roc_data = [FPR, TPR, AUC]
    # prc_data = [recall, precision, AP]
    return performance, roc_data, prc_data

# draw ROC
def ROC(fpr, tpr, roc_auc):
    plt.figure()
    lw = 2
    plt.figure(figsize=(10, 10))
    plt.plot(fpr, tpr, color='darkorange',
             lw=lw, label='ROC curve (area = %0.2f)' % roc_auc)
    plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate', fontdict={'weight': 'normal', 'size': 30})
    plt.ylabel('True Positive Rate', fontdict={'weight': 'normal', 'size': 30})
    plt.title('Receiver operating characteristic github_project', fontdict={'weight': 'normal', 'size': 30})
    plt.legend(loc="lower right", prop={'weight': 'normal', 'size': 30})
    plt.show()

# draw PRC
def PRC(recall, precision, AP):
    plt.figure()
    plt.step(recall, precision, color='b', alpha=0.2,
             where='post')
    plt.fill_between(recall, precision, step='post', alpha=0.2,
                     color='b')

    plt.xlabel('Recall')
    plt.ylabel('Precision')
    plt.ylim([0.0, 1.05])
    plt.xlim([0.0, 1.05])
    plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format(AP))
    plt.show()


"""
Module for querying the reference and test products that will be used
in the intercomparison evaluation.
As the test or reference datasets might be missing some records, we
need to perform an inner join such that there is a 1-1 relationship.
Product names can be different, database environments can be different;
However, both must have their level-1 ancestor uuid available in the
yaml document for a given dataset. If not, that dataset record will be
removed from the resulting query.
"""

from pathlib import Path
from typing import Any, Dict, Optional, Tuple
import datacube  # type: ignore
import pandas  # type: ignore
import structlog  # type: ignore

from gost.odc_documents import load_odc_metadata

_LOG = structlog.get_logger()

def query_db(
    env: str,
    product_name: str,
    time: Optional[Tuple[str, str]] = None,
    lon: Optional[Tuple[str, str]] = None,
    lat: Optional[Tuple[str, str]] = None,
    additional_filters: Optional[Dict[str, Any]] = None,
) -> pandas.DataFrame:
    """
    Generic datacube query wrapper.

    :param env:
        Name of the database environment to query.

    :param product_name:
        Name of the product to query.

    :param time:
        The time (earliest, latest) extents (optional).

    :param lon:
        The longitude (left, right) extents (optional).

    :param lat:
        The latitude (top, bottom) extents (optional).

    :param additional_filters:
        A dictionary containing {field: value} pairs that datacube can
        use to further filter datasets (optional).
        e.g. {"region_code": "092084"}
    """

    dc = datacube.Datacube(env=env)

    _LOG.info(
        "finding datasets",
        env=env,
        product_name=product_name,
        time=time,
        lon=lon,
        lat=lat,
    )
    datasets = dc.find_datasets(
        product_name, time=time, lon=lon, lat=lat, **additional_filters
    )

    uuid = []
    yaml_pathname = []
    proc_info_pathname = []

    for dataset in datasets:
        _LOG.info("processing dataset", dataset=str(dataset.local_path))
        doc = load_odc_metadata(dataset.local_path)

        uuid.append(doc.parent_uuid)
        yaml_pathname.append(str(dataset.local_path))

        # procssing info document
        proc_info_pathname = dataset.local_path.parent.joinpath(doc.proc_info)
        proc_info_pathname.append(str(proc_info_pathname))

    dataframe = pandas.DataFrame(
        {
            "level1_uuid": uuid,
            "yaml_pathname": yaml_pathname,
            "proc_info_pathname": proc_info_pathname,
        }
    )

    return dataframe

def query_products(
    product_name_test: str,
    product_name_reference: str,
    db_env_test: str,
    db_env_reference: str,
    time: Optional[Tuple[str, str]] = None,
    lon: Optional[Tuple[str, str]] = None,
    lat: Optional[Tuple[str, str]] = None,
    additional_filters: Optional[Dict[str, Any]] = None,
) -> pandas.DataFrame:
    """
    Queries an ODC Database for both the test and reference products,
    then merges the results based on a common ancestor uuid.
    Assumes products have been indexed with lineage.

    :param product_name_test:
        Product name for the test datasets.

    :param product_name_reference:
        Product name for the reference datasets.

    :param db_env_test:
        Name of the database environment containing the test datasets.

    :param db_env_reference:
        Name of the database environment containing the reference
        datasets.

    :param time:
        The time (earliest, latest) extents (optional).

    :param lon:
        The longitude (left, right) extents (optional).

    :param lat:
        The latitude (top, bottom) extents (optional).

    :param additional_filters:
        A dictionary containing {field: value} pairs that datacube can
        use to further filter datasets.
        e.g. {"region_code": "092084"}
    """

    _LOG.info("querying the test datasets")
    test_dataframe = query_db(
        db_env_test, product_name_test, time, lon, lat, **additional_filters
    )

    _LOG.info("querying the reference datasets")
    reference_dataframe = query_db(
        db_env_reference, product_name_reference, time, lon, lat, **additional_filters
    )

    _LOG.info("filtering test and reference datasets for common ancestor")
    merged = pandas.merge(
        test_dataframe,
        reference_dataframe,
        on="uuid",
        how="inner",
        suffixes=["_test", "_reference"],
    )

    return merged

def query_filepath(path: Path, pattern: str) -> pandas.DataFrame:
    """
    Find datasets by globbing the filesystem.

    :param path:
        The full pathname to the directory containing the product.

    :param pattern:
        A string containing pattern used to glob the filesystem.
        eg `*/*/2019/05/*/*.odc-metadata.yaml`
    """
    files = list(path.rglob(pattern))

    uuid = []
    yaml_pathname = []
    proc_info_pathname = []

    _LOG.info(
        "finding datasets",
        path=path,
        pattern=pattern,
    )

    for fname in files:
        _LOG.info("processing dataset", dataset=str(fname))

        doc = load_odc_metadata(fname)
        uuid.append(doc.parent_uuid)
        yaml_pathname.append(str(fname))

        # procssing info document
        pathname = fname.parent.joinpath(doc.proc_info)
        proc_info_pathname.append(str(pathname))

    dataframe = pandas.DataFrame(
        {
            "level1_uuid": uuid,
            "yaml_pathname": yaml_pathname,
            "proc_info_pathname": proc_info_pathname,
        }
    )

    return dataframe

def query_via_filepath(
    test_directory: str,
    reference_directory: str,
    test_pattern: str,
    reference_pattern: str,
) -> pandas.DataFrame:
    """
    Find the test and reference datasets by globbing the filesystem.

    :param test_directory:
        The full pathname to the directory containing the test product.

    :param reference_directory:
        The full pathname to the directory containing the reference product.

    :param test_pattern:
        A string containing pattern used to glob the filesystem.
        eg `*/*/2019/05/*/*.odc-metadata.yaml`

    :param reference_pattern:
        A string containing pattern used to glob the filesystem.
        eg `*/*/2019/05/*/*.odc-metadata.yaml`
    """

    test_dir = Path(test_directory)
    reference_dir = Path(reference_directory)

    _LOG.info("querying the test datasets")
    test_dataframe = query_filepath(test_dir, test_pattern)

    _LOG.info("querying the reference datasets")
    reference_dataframe = query_filepath(reference_dir, reference_pattern)

    _LOG.info("filtering test and reference datasets for common ancestor")
    merged = pandas.merge(
        test_dataframe,
        reference_dataframe,
        on="level1_uuid",
        how="inner",
        suffixes=["_test", "_reference"],
    )

    return merged


from datetime import timedelta

from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers

from .models import Match

class MatchSerializer(serializers.ModelSerializer):
    class Meta:
        model = Match
        fields = (
            "id",
            "start_at",
            "finished_at",
            "game",
            "home_robot",
            "away_robot",
            "home_score",
            "away_score",
            "runner_log",
            "created_at",
            "updated_at",
        )
        read_only_fields = (
            "id",
            "finished_at",
            "game",
            "home_score",
            "away_score",
            "runner_log",
            "created_at",
            "updated_at",
        )

    def validate_home_robot(self, value):
        if value.owner != self.context["request"].user:
            raise serializers.ValidationError(
                _("home_robot must be one of your robots.")
            )
        return value

    def validate_away_robot(self, value):
        if value.owner == self.context["request"].user:
            raise serializers.ValidationError(
                _("away_robot must not be one of your robots.")
            )
        return value

    def validate_start_at(self, value):
        if value < timezone.now() + timedelta(minutes=1):
            raise serializers.ValidationError(
                _("start_at must be at least 1 minute from now.")
            )
        return value

    def validate(self, data):
        if data["home_robot"].game != data["away_robot"].game:
            raise serializers.ValidationError(
                _("home_robot and away_robot must play the same game")
            )
        data["game"] = data["home_robot"].game
        return data


#!/usr/bin/env python

import sys, pysam, re, subprocess
import numpy
import my_seq

SAre = re.compile('([^ \t\n\r\f\v,]+),(\d+),([\-\+]),(\w+),(\d+),(\d+);')
cigarMDRe = re.compile('(\d+)([MD])')
cigarHIMSRe = re.compile('(\d+)([HIMS])')
cigarHSRe_right = re.compile('(\d+)([HS])$')
cigarHSRe_left = re.compile('^(\d+)([HS])')

def parse_bp_from_bam(input_bam, output_file, key_seq_size, min_major_clip_size, max_minor_clip_size):

    """
    function for getting breakpoints from BAM file
    input:bam file
    output:output_file + ".bp.tmp.txt"
    breakpoint info (juncChr, juncPos-1, juncPos, Dir (right clipping:+, left clipping:-), Juncseq, ID + ("/1" or "/2"), MAPQ, Clipping size, AlignmentSize, Base quality of juncseq)
    """

    bamfile = pysam.Samfile(input_bam, "rb")
    hout = open(output_file, "w")

    # maybe add the regional extraction of bam files
    for read in bamfile.fetch():

        # get the flag information
        flags = format(int(read.flag), "#014b")[:1:-1]

        # skip if not aligned
        if flags[2] == "1": continue

        # skip supplementary alignment
        if flags[8] == "1" or flags[11] == "1": continue

        # skip duplicated reads
        if flags[10] == "1": continue

        # no clipping
        if len(read.cigar) == 1: continue

        # get the clipping size in the both side
        left_clipping = (read.cigar[0][1] if read.cigar[0][0] in [4, 5] else 0)
        right_clipping = (read.cigar[len(read.cigar) - 1][1] if read.cigar[len(read.cigar) - 1][0] in [4, 5] else 0)

        if left_clipping < min_major_clip_size and right_clipping < min_major_clip_size: continue

        # get the alignment basic information
        chr_current = bamfile.getrname(read.tid)
        pos_current = int(read.pos + 1)
        dir_current = ("-" if flags[4] == "1" else "+")

        # when the right side is clipped...
        if right_clipping >= min_major_clip_size:
            clipLen_current = right_clipping
            alignmentSize_current = read.alen
            readLength_current = read.rlen
            juncChr_current = chr_current
            juncPos_current = pos_current + alignmentSize_current - 1
            juncDir_current = "+"
            juncseq_start = readLength_current - clipLen_current
            juncseq_end = readLength_current - clipLen_current + key_seq_size
            juncseq = read.seq[juncseq_start:juncseq_end]

            #filter if base qualities of junction seq is low
            #if numpy.mean(read.query_qualities[juncseq_start:juncseq_end]) < 10:
            #    continue

            print >> hout, '\t'.join([juncChr_current, str(juncPos_current-1), str(juncPos_current), juncDir_current, juncseq,
                                      read.qname + ("/1" if flags[6] == "1" else "/2"), str(read.mapq), str(right_clipping), str(alignmentSize_current)])

        if left_clipping >= min_major_clip_size:

            clipLen_current = left_clipping
            alignmentSize_current = read.alen
            readLength_current = read.rlen

            juncChr_current = chr_current
            juncPos_current = pos_current
            juncDir_current = "-"

            juncseq_end = clipLen_current
            juncseq_start = clipLen_current - key_seq_size
            juncseq = my_seq.reverse_complement(read.seq[juncseq_start:juncseq_end])

            #filter if base qualities of soft clipping part is low
            #if numpy.mean(read.query_qualities[juncseq_start:juncseq_end])<10:
            #  continue

            print >> hout, '\t'.join([juncChr_current, str(juncPos_current-1), str(juncPos_current), juncDir_current, juncseq,
                                      read.qname + ("/1" if flags[6] == "1" else "/2"), str(read.mapq), str(left_clipping), str(alignmentSize_current)])

    bamfile.close()
    hout.close()

def cluster_breakpoint(input_file, output_file, check_interval):

    """
    function for clustering breakpoints
    input:output_file + ".bp.tmp.txt"               result of "parse_bp_from_bam" function
    output:output_file + ".bp.clustered.tmp.txt"    breakpoint info (juncChr, juncPos-1, juncPos, Dir (right clipping:+, left clipping:-), Juncseq, IDs + ("/1" or "/2"), MAPQs, Clipping sizes, AlignmentSizes, Base quality of juncseq)
    """

    hout = open(output_file, 'w')

    tmp_chr = ""
    tmp_pos = 0

    key2juncseq = {}
    key2read = {}
    key2mapq = {}
    key2clipsize = {}
    key2alnsize = {}
    #key2baseq = {}

    with open(input_file, 'r') as hin:
        for line in hin:
            F = line.rstrip('\n').split('\t')

            if tmp_chr != F[0] or int(F[1]) - tmp_pos > check_interval:
                del_list = []
                for key in key2read:
                    print >> hout, key + '\t' + ';'.join(list(set(key2juncseq[key]))) + '\t' + ';'.join(key2read[key]) + '\t' + ';'.join(key2mapq[key]) + '\t' + \
                                   ';'.join(key2clipsize[key]) + '\t' + ';'.join(key2alnsize[key])
                    del_list.append(key)

                for key in del_list:
                    del key2juncseq[key]
                    del key2read[key]
                    del key2mapq[key]
                    del key2clipsize[key]
                    del key2alnsize[key]
                    #del key2baseq[key]

                tmp_chr = F[0]
                tmp_pos = int(F[1])

            key = F[0] + '\t' + F[1] + '\t' + F[2] + '\t' + F[3]
            if key not in key2juncseq: key2juncseq[key] = []
            if key not in key2read: key2read[key] = []
            if key not in key2mapq: key2mapq[key] = []
            if key not in key2clipsize: key2clipsize[key] = []
            if key not in key2alnsize: key2alnsize[key] = []
            #if key not in key2baseq: key2baseq[key] = []

            key2juncseq[key].append(F[4])
            key2read[key].append(F[5])
            key2mapq[key].append(F[6])
            key2clipsize[key].append(F[7])
            key2alnsize[key].append(F[8])
            #key2baseq[key].append(F[9])

    # final flush
    for key in key2read:
        print >> hout, key + '\t' + ';'.join(list(set(key2juncseq[key]))) + '\t' + ';'.join(key2read[key]) + '\t' + ';'.join(key2mapq[key]) + '\t' + \
                       ';'.join(key2clipsize[key]) + '\t' + ';'.join(key2alnsize[key])

    hout.close()


from django.contrib import admin
from django.urls import path, include
#from . import views
from . views import (
    PostListView,
    PostDetailView,
    PostCreateView,
    PostUpdateView,
    PostDeleteView,
    PostSearchView
)

'''urlpatterns = [
    path('', views.home,name='home'),
]'''
urlpatterns = [
    #path('',views.home,name='home')
    path('',PostListView.as_view(),name='home'),
    path('post/<int:pk>/', PostDetailView.as_view(), name='post-detail'),
    path('post/new',PostCreateView.as_view(),name='post-create'),
    path('post/<int:pk>/update/',PostUpdateView.as_view(),name='post-update'),
    path('post/<int:pk>/delete/',PostDeleteView.as_view(),name='post-delete'),
    path('search',PostSearchView.as_view(),name='search')

]


from flask import Blueprint, render_template, current_app , session

from web_app.routes.wrappers import authenticated_route
from web_app.routes.forms.create_funding_form import CreateFundingForm

funding_routes = Blueprint("funding_routes", __name__)

@funding_routes.route("/create-funding", methods=['GET'])
@authenticated_route
def create_funding():
    service = current_app.config["FIREBASE_SERVICE"]
    fund_types = service.fetch_fund_types()
    form = CreateFundingForm()
    form.fundTypeSelect.choices = [(ft['type'], ft['type'].lower().replace("_"," ")) for ft in fund_types]
    return render_template("create_funding.html", form=form, fund_types=fund_types)

@funding_routes.route("/create-funding", methods=["POST"])
def handle_create_funding():
    form = CreateFundingForm()
    print("FORM: ", form)
    if form.validate_on_submit():
        title = form.fundraiserTitle.data
        print("TITLE: ", title)


from django.db import models
Choices=[
    ("Select", "Select"),

    ("Entertinment",
     (
         ("Cinema", "Cinema"),
     ("Stories", "Stories"),
     ("Others", "Others"),
     )
     ),
    ("Business", "Business"),
    ("Sports", "Sports"),
    ("Education", "Education"),
    ("Careers",
      (
         ("IT jobs", "IT Jobs"),
         ("Non-IT jobs", "Non-IT Jobs"),
         ("Government jobs", "Government jobs"),
         ("Others", "Others"),
      )
    ),
    ("Hot News", "Hot News"),
]
# Create your models here.
class NewsModel(models.Model):
    news_type = models.CharField(max_length=50, choices=Choices, default="Select")
    news_headline = models.CharField(max_length=50)
    post_news_text = models.CharField(max_length=500)
    post_news_image = models.FileField(upload_to="app_news/static/Images",null=True, blank=True)
    def __str__(self):
        return self.news_type


#!/usr/bin/env python
# -*- coding: utf-8 -*-
###################################################################
# Author: Mu yanru
# Date  : 2019.2
# Email : muyanru345@163.com
###################################################################

import functools

from dayu_widgets.MCheckBox import MCheckBox
from dayu_widgets.MMenu import MMenu
from dayu_widgets.MPushButton import MPushButton
from dayu_widgets.MRadioButton import MRadioButton
from dayu_widgets.MToolButton import MToolButton
from dayu_widgets.mixin import property_mixin
from dayu_widgets.qt import *

@property_mixin
class MButtonGroupBase(QWidget):
    def __init__(self, orientation=Qt.Horizontal, parent=None):
        super(MButtonGroupBase, self).__init__(parent=parent)
        self._main_layout = QBoxLayout(
            QBoxLayout.LeftToRight if orientation == Qt.Horizontal else QBoxLayout.TopToBottom)
        self._main_layout.setContentsMargins(0, 0, 0, 0)
        self.setLayout(self._main_layout)
        self.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Minimum)
        self._button_group = QButtonGroup()
        self._orientation = 'horizontal' if orientation == Qt.Horizontal else 'vertical'

    def set_spacing(self, value):
        self._main_layout.setSpacing(value)

    def get_button_group(self):
        return self._button_group

    def create_button(self, data_dict):
        raise NotImplementedError()

    def add_button(self, data_dict, index=None):
        if isinstance(data_dict, basestring):
            data_dict = {'text': data_dict}
        elif isinstance(data_dict, QIcon):
            data_dict = {'icon': data_dict}
        button = self.create_button(data_dict)
        button.setProperty('combine', self._orientation)
        if data_dict.get('data'):
            button.setProperty('data', data_dict.get('data'))
        if data_dict.get('checked'):
            button.setProperty('checked', data_dict.get('checked'))
        if data_dict.get('shortcut'):
            button.setProperty('shortcut', data_dict.get('shortcut'))
        if data_dict.get('tooltip'):
            button.setProperty('toolTip', data_dict.get('tooltip'))
        if data_dict.get('clicked'):
            button.clicked.connect(data_dict.get('clicked'))
        if data_dict.get('toggled'):
            button.toggled.connect(data_dict.get('toggled'))

        if index is None:
            self._button_group.addButton(button)
        else:
            self._button_group.addButton(button, index)
        self._main_layout.insertWidget(self._main_layout.count(), button)

    def set_button_list(self, button_list):
        for button in self._button_group.buttons():
            self._button_group.removeButton(button)
            self._main_layout.removeWidget(button)
            button.setVisible(False)
        for index, data_dict in enumerate(button_list):
            self.add_button(data_dict, index)

class MPushButtonGroup(MButtonGroupBase):
    def __init__(self, size=None, type=None, orientation=Qt.Horizontal, parent=None):
        super(MPushButtonGroup, self).__init__(orientation=orientation, parent=parent)
        self.set_spacing(1)
        self._type = type
        self._size = size
        self._button_group.setExclusive(False)

    def create_button(self, data_dict):
        return MPushButton(text=data_dict.get('text'),
                           icon=data_dict.get('icon'),
                           size=data_dict.get('size') or self._size,
                           type=data_dict.get('type') or self._type,
                           parent=self
                           )

class MCheckBoxGroup(MButtonGroupBase):
    sig_checked_changed = Signal(list)

    def __init__(self, orientation=Qt.Horizontal, parent=None):
        super(MCheckBoxGroup, self).__init__(orientation=orientation, parent=parent)
        self.set_spacing(15)
        self._button_group.setExclusive(False)

        self.setContextMenuPolicy(Qt.CustomContextMenu)
        self.customContextMenuRequested.connect(self._slot_context_menu)

        self._button_group.buttonClicked[int].connect(self._slot_map_signal)
        self.set_value([])

    def create_button(self, data_dict):
        return MCheckBox(text=data_dict.get('text'),
                         icon=data_dict.get('icon'),
                         parent=self
                         )

    @Slot(QPoint)
    def _slot_context_menu(self, point):
        context_menu = MMenu(parent=self)
        action_select_all = context_menu.addAction('Select All')
        action_select_none = context_menu.addAction('Select None')
        action_select_invert = context_menu.addAction('Select Invert')
        action_select_all.triggered.connect(functools.partial(self._slot_set_select, True))
        action_select_none.triggered.connect(functools.partial(self._slot_set_select, False))
        action_select_invert.triggered.connect(functools.partial(self._slot_set_select, None))
        context_menu.exec_(QCursor.pos() + QPoint(10, 10))

    @Slot(bool)
    def _slot_set_select(self, state):
        for check_box in self._button_group.buttons():
            if state is None:
                old_state = check_box.isChecked()
                check_box.setChecked(not old_state)
            else:
                check_box.setChecked(state)
        self._slot_map_signal()

    @Slot(int)
    def _slot_map_signal(self, state=None):
        self.sig_checked_changed.emit(
            [check_box.text() for check_box in self._button_group.buttons() if check_box.isChecked()])

    def set_value(self, value):
        if not isinstance(value, list):
            value = [value]
        self.setProperty('value', value)

    def _set_value(self, value):
        edit_from_code = False
        for check_box in self._button_group.buttons():
            flag = Qt.Checked if check_box.text() in value else Qt.Unchecked
            if flag != check_box.checkState():
                # 更新来自代码
                edit_from_code = True
                check_box.setCheckState(flag)
        if edit_from_code:
            self.sig_checked_changed.emit(value)

class MRadioButtonGroup(MButtonGroupBase):
    sig_checked_changed = Signal(int)

    def __init__(self, orientation=Qt.Horizontal, parent=None):
        super(MRadioButtonGroup, self).__init__(orientation=orientation, parent=parent)
        self.set_spacing(15)
        self._button_group.setExclusive(True)
        self._button_group.buttonClicked[int].connect(functools.partial(self.setProperty, 'checked'))
        self._button_group.buttonClicked[int].connect(self.sig_checked_changed)
        self.set_checked(-1)

    def create_button(self, data_dict):
        return MRadioButton(text=data_dict.get('text'),
                            icon=data_dict.get('icon'),
                            parent=self
                            )

    def _set_checked(self, value):
        assert isinstance(value, int)
        if value != self._button_group.checkedId():
            # 更新来自代码
            button = self._button_group.button(value)
            if button:
                button.setChecked(True)
            self.sig_checked_changed.emit(value)

    def set_checked(self, value):
        self.setProperty('checked', value)

class MToolButtonGroup(MButtonGroupBase):
    sig_checked_changed = Signal(int)

    def __init__(self, size=None, type=None, exclusive=False, orientation=Qt.Horizontal, parent=None):
        super(MToolButtonGroup, self).__init__(orientation=orientation, parent=parent)
        self.set_spacing(1)
        self._button_group.setExclusive(exclusive)
        self._size = size
        self._type = type
        self._button_group.buttonClicked[int].connect(self.set_checked)
        self._button_group.buttonClicked[int].connect(self.sig_checked_changed)
        self.set_checked(-1)

    def create_button(self, data_dict):
        button = MToolButton(text=data_dict.get('text'),
                             icon=data_dict.get('icon'),
                             type=data_dict.get('type') or self._type,
                             size=data_dict.get('size') or self._size,
                             parent=self
                             )
        button.setCheckable(data_dict.get('checkable', False))
        if data_dict.get('icon_checked'):
            button.setProperty('icon_checked', data_dict.get('icon_checked'))
            button.setProperty('icon_unchecked', data_dict.get('icon'))
        return button

    def _set_checked(self, value):
        assert isinstance(value, int)
        if value != self._button_group.checkedId():
            # 更新来自代码
            button = self._button_group.button(value)
            if button:
                button.setChecked(True)
            self.sig_checked_changed.emit(value)

    def set_checked(self, value):
        self.setProperty('checked', value)


import torch

#########################################################
# Operation List:
#   1. gather
#   2. radius gaussian
#   3. closest_pool
#   4. max_pool
#   5. global_average
#########################################################

########################################
# 1. gather
########################################

def gather(x, idx, method=2):
    """
    implementation of a custom gather operation for faster backwards.
    :param x: input with shape [N, D_1, ... D_d]
    :param idx: indexing with shape [n_1, ..., n_m]
    :param method: Choice of the method
    :return: x[idx] with shape [n_1, ..., n_m, D_1, ... D_d]
    """

    if method == 0:
        return x[idx]
    elif method == 1:
        x = x.unsqueeze(1)
        x = x.expand((-1, idx.shape[-1], -1))
        idx = idx.unsqueeze(2)
        idx = idx.expand((-1, -1, x.shape[-1]))
        return x.gather(0, idx)
    elif method == 2:
        for i, ni in enumerate(idx.size()[1:]):
            x = x.unsqueeze(i+1)
            new_s = list(x.size())
            new_s[i+1] = ni
            x = x.expand(new_s)
        n = len(idx.size())
        for i, di in enumerate(x.size()[n:]):
            idx = idx.unsqueeze(i+n)
            new_s = list(idx.size())
            new_s[i+n] = di
            idx = idx.expand(new_s)
        return x.gather(0, idx)
    else:
        raise ValueError('Unkown method')

########################################
# 2. radius_gaussian
########################################

def radius_gaussian(sq_r, sig, eps=1e-9):
    """
    Compute a radius gaussian (gaussian of distance)
    :param sq_r: input radiuses [dn, ..., d1, d0]
    :param sig: extents of gaussians [d1, d0] or [d0] or float
    :return: gaussian of sq_r [dn, ..., d1, d0]
    """
    return torch.exp(-sq_r / (2 * sig**2 + eps))

########################################
# 3. closest_pool
########################################

def closest_pool(x, inds):
    """
    Pools features from the closest neighbors. WARNING: this function assumes the neighbors are ordered.
    :param x: [n1, d] features matrix
    :param inds: [n2, max_num] Only the first column is used for pooling
    :return: [n2, d] pooled features matrix
    """

    # Add a last row with minimum features for shadow pools
    x = torch.cat((x, torch.zeros_like(x[:1, :])), 0)

    # Get features for each pooling location [n2, d]
    return gather(x, inds[:, 0])

########################################
# 4. max_pool
########################################

def max_pool(x, inds):
    """
    Pools features with the maximum values.
    :param x: [n1, d] features matrix
    :param inds: [n2, max_num] pooling indices
    :return: [n2, d] pooled features matrix
    """

    # Add a last row with minimum features for shadow pools
    x = torch.cat((x, torch.zeros_like(x[:1, :])), 0)

    # Get all features for each pooling location [n2, max_num, d]
    pool_features = gather(x, inds)

    # Pool the maximum [n2, d]
    max_features, _ = torch.max(pool_features, 1)
    return max_features

########################################
# 5. max_pool
########################################

def global_average(x, batch_lengths):
    """
    Block performing a global average over batch pooling
    :param x: [N, D] input features
    :param batch_lengths: [B] list of batch lengths
    :return: [B, D] averaged features
    """

    # Loop over the clouds of the batch
    averaged_features = []
    i0 = 0
    for b_i, length in enumerate(batch_lengths):

        # Average features for each batch cloud
        averaged_features.append(torch.mean(x[i0:i0 + length], dim=0))

        # Increment for next cloud
        i0 += length

    # Average features in each batch
    return torch.stack(averaged_features)


"""Tests for dataset functions."""

import unittest

from ..data_processing.dataset import load_dataset
from ..critical_success_factors import ALL_CSFS
from ..success_metrics import ALL_SUCCESS_METRICS
from ..utils import extract_csf_values, extract_success_values

class TestDataset(unittest.TestCase):
    """Tests for dataset functions."""

    def test_dataset_load(self):
        """Test loading the dataset."""
        data = load_dataset()

        # There should be 101 projects in the dataset
        self.assertEqual(len(data), 101)

        # Each project should have 42 columns
        len_success_metrics = len(data[0][1])
        len_critical_success_factors = len(data[0][0])
        total_columns = len_critical_success_factors + len_success_metrics

        expected_columns = len(ALL_CSFS) + len(ALL_SUCCESS_METRICS)

        self.assertEqual(total_columns, expected_columns)

    def test_dataset_format(self):
        """Test the format of the output of the dataset."""
        data = load_dataset()

        critical_success_factors, success_metrics = data[0]

        extracted_csf_values = extract_csf_values(critical_success_factors)
        extracted_sm_values = extract_success_values(success_metrics)

        # These are ordered by the order of the CSFs and SuccessMetrics
        actual_csf_values = [4, 1, 5, 5, 5, 5]  # OF
        actual_csf_values += [5, 5, 5, 5, 5, 5, 4]  # TF
        actual_csf_values += [3, 3, 3, 4]  # CF
        actual_csf_values += [5, 5, 4, 5, 3, 5, 5]  # PF

        actual_sm_values = [5, 4, 5, 5]  # Process
        actual_sm_values += [5, 5, 5, 5, 5, 5, 5, 5, 5, 2]  # Product
        actual_sm_values += [5, 5, 5]  # Stakeholder
        actual_sm_values += [4.76]  # Overall

        self.assertEqual(extracted_csf_values, actual_csf_values)
        self.assertEqual(extracted_sm_values, actual_sm_values)

if __name__ == '__main__':
    unittest.main()


'''
Cálculo de areas en base a clusterización utilizando k-means, en imagenes satelite
parametros:
-- file imagen satelite
-- area area en km2 representada en la imagen satelite
se utilizan 3 clusters por imagen, (conversion a niveles de gris) con salto amplio
en la polarización de la imagen

MIT License
'''
import numpy as np
import matplotlib.pyplot as plt
from PIL import Image
from sklearn.cluster import KMeans
import argparse
# construct the argument parser and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-f", "--file", required=True,
                help="satellite image")
ap.add_argument("-a", "--area", required=True,
	help="satellite image area in km2")

args = vars(ap.parse_args())
# open file name
filename = args["file"]
area = args["area"]
I = Image.open(filename)

#resize and paint with axis
plt.figure(figsize=(8,8))
plt.imshow(I)
plt.axis('on')
plt.show()

# paint in gray scale
I1 = I.convert('L')
I2 = np.asarray(I1,dtype=np.float)

plt.figure(figsize=(8,8))
plt.imshow(I2,cmap='gray')
plt.axis('on')
plt.show()

# define 3 clusters and grey scale (black, white, mix scale)
X = I2.reshape((-1, 1))
k_means = KMeans(n_clusters=3)
'''
k_means=KMeans(algorithm='auto', copy_x=True, init='k-means++', max_iter=300,
    n_clusters=3, n_init=10, n_jobs=1, precompute_distances='auto',
    random_state=None, tol=0.0001, verbose=0)
'''

k_means.fit(X)

centroides = k_means.cluster_centers_
etiquetas = k_means.labels_

I2_compressed = np.choose(etiquetas, centroides)
I2_compressed.shape = I2.shape

plt.figure(figsize=(8,8))
plt.imshow(I2_compressed,cmap='gray')
plt.axis('on')
plt.show()

I2 = (I2_compressed-np.min(I2_compressed))/(np.max(I2_compressed)-np.min(I2_compressed))*255
I2 = Image.fromarray(I2.astype(np.uint8))
w, h =I2.size
colors = I2.getcolors(w * h)
print('density points:',colors)

print (u'Área = ',  float(area)*float(colors[0][0])/float(w*h), 'km2')


import itertools
import math
import os.path
import re
import time

from collections import defaultdict, Counter

def monad(num, steps):
    variables = defaultdict(int)
    counter = 0
    for step in steps:
        if step[0] == "inp":
            variables[step[1]] = int(num[counter])
            counter += 1
        else:
            first = variables[step[1]]
            second = (
                int(step[2]) if step[2].lstrip("-").isdigit() else variables[step[2]]
            )
            if step[0] == "mul":
                variables[step[1]] *= second
            elif step[0] == "eql":
                variables[step[1]] = 1 if first == second else 0
            elif step[0] == "add":
                variables[step[1]] += second
            elif step[0] == "div":
                if second == 0:
                    return 1
                variables[step[1]] //= second
            elif step[0] == "mod":
                if variables[step[1]] < 0 or second <= 0:
                    return 1
                variables[step[1]] %= second
    print(num)
    print(variables)
    return variables["z"]

def part1(steps):
    for num in range(11111111111111, 99999999999999):
        # for num in range(99999999999999,11111111111110,-1):
        n = str(num)
        if not "0" in n:
            res = monad(n, steps)
            if not res:
                break
    return num

if __name__ == "__main__":
    replit_start_time = time.perf_counter()
    counter = 0
    for i in range(1, 717):
        counter += len(str(i))
        print(i, counter)
        if counter >= 2040:
            print(i, counter)
            break
    print(counter)

    with open(os.path.join(os.path.dirname(__file__), "data.txt")) as f:
        data = [d.split() for d in f.read().splitlines()]
        # print(monad(str(99999999927392-7645130729),data))
        print(sum(int(digit) for digit in str(11111111119195)))

        # print(part1(data))
    # Keep this line at the end of your code
    replit_end_time = time.perf_counter()
    print("Elapsed time:", replit_end_time - replit_start_time)


import os
import json
import socket
import time

import utils

from front_base.ip_manager import IpManagerBase

class IpManager(IpManagerBase):
    def __init__(self, config, default_domain_fn, domain_fn, speed_fn, logger):
        super(IpManager, self).__init__(config, None, logger, speed_fn)
        self.default_domain_fn = default_domain_fn
        self.domain_fn = domain_fn

        self.domain_map = self.load_domains()
        # top_domain -> {}

    def __str__(self):
        o = ""
        o += " domain_map: \r\n%s\r\n" % json.dumps(self.domain_map, indent=2)
        return o

    def load_domains(self):
        domain_map = {}  # top_domain -> {}
        for fn in [self.domain_fn, self.default_domain_fn]:
            if not os.path.isfile(fn):
                continue

            try:
                with open(fn, "r") as fd:
                    ds = json.load(fd)
                    for top in ds:
                        domain_map[str(top)] = {
                            "links": 0,
                            "fail_times": 0,
                            "last_try": 0.0
                        }
                self.logger.info("load %s success", fn)
                break
            except Exception as e:
                self.logger.warn("load %s for host failed:%r", fn, e)
        return domain_map

    def save_domains(self, domains):
        ns = []
        for top_domain, _ in self.domain_map.items():
            ns.append(top_domain)

        if ns == domains:
            self.logger.debug("save domains not changed, ignore")
            return
        else:
            self.logger.info("save domains:%s", domains)

        dat = {}
        for domain in domains:
            dat[domain] = ["www." + domain]

        with open(self.domain_fn, "w") as fd:
            json.dump(dat, fd)

        self.domain_map = self.load_domains()

    def get_ip_sni_host(self):
        now = time.time()
        for top_domain, info in self.domain_map.items():
            if info["links"] < 0:
                info["links"] = 0

            if info["links"] >= self.config.max_connection_per_domain:
                continue

            if info["fail_times"] and now - info["last_try"] < 60:
                continue

            sni = "www." + top_domain
            try:
                ip = socket.gethostbyname(sni)
            except Exception as e:
                self.logger.warn("get ip for %s fail:%r", sni, e)
                continue

            self.logger.debug("get ip:%s sni:%s", ip, sni)
            info["links"] += 1
            info["last_try"] = now
            return ip, sni, top_domain

        return None, None, None

    def _get_domain(self, top_domain):
        self.domain_map.setdefault(top_domain, {
                            "links": 0,
                            "fail_times": 0,
                            "last_try": 0.0
                        })
        return self.domain_map[top_domain]

    def report_connect_fail(self, ip_str, sni=None, reason="", force_remove=True):
        ip, _ = utils.get_ip_port(ip_str)
        ip = utils.to_str(ip)
        top_domain = ".".join(sni.split(".")[1:])

        info = self._get_domain(top_domain)
        info["fail_times"] += 1
        info["links"] -= 1
        self.logger.debug("ip %s sni:%s connect fail, reason:%s", ip, sni, reason)

    def update_ip(self, ip_str, sni, handshake_time):
        top_domain = ".".join(sni.split(".")[1:])

        info = self._get_domain(top_domain)
        info["fail_times"] = 0
        info["last_try"] = 0.0
        # self.logger.debug("ip %s sni:%s connect success, rtt:%f", ip, sni, handshake_time)

    def ssl_closed(self, ip_str, sni=None, reason=""):
        ip, _ = utils.get_ip_port(ip_str)
        ip = utils.to_str(ip)
        top_domain = ".".join(sni.split(".")[1:])

        try:
            info = self._get_domain(top_domain)
            info["links"] -= 1
            self.logger.debug("ip %s sni:%s connect closed reason %s", ip, sni, reason)
        except Exception as e:
            self.logger.warn("ssl_closed %s sni:%s reason:%s except:%r", ip_str, sni, reason, e)


import asyncpg
from aiogram.dispatcher import FSMContext
from aiogram.dispatcher.filters import Text
from aiogram.types import Message, ReplyKeyboardRemove, InlineKeyboardMarkup, InlineKeyboardButton, CallbackQuery

from keyboards.default import menu
from keyboards.inline.callback_data import select_dictionary_callback, add_translate_callback
from loader import dp, db
from states import CreateNewTranslate

@dp.callback_query_handler(add_translate_callback.filter(is_selected='True'))
async def new_translate_callback(call: CallbackQuery):
    """Query for input word"""
    await call.answer()
    await call.message.delete()
    await call.message.answer("Напиши слово на английском", reply_markup=ReplyKeyboardRemove())
    await CreateNewTranslate.SetEnglishWord.set()

@dp.message_handler(Text("📌Новый перевод"))
async def new_translate(message: Message):
    """Query for input word"""
    await message.answer("Напиши слово на английском", reply_markup=ReplyKeyboardRemove())
    await CreateNewTranslate.SetEnglishWord.set()

@dp.message_handler(state=CreateNewTranslate.SetEnglishWord)
async def set_english_word(message: Message, state: FSMContext):
    """Query for input translate for word"""
    await state.update_data(english_word=message.text)
    await message.answer("Теперь введи перевод своего слова")
    await CreateNewTranslate.SetRussianWord.set()

@dp.message_handler(state=CreateNewTranslate.SetRussianWord)
async def set_translate(message: Message, state: FSMContext):
    await state.update_data(russian_word=message.text)
    data = await state.get_data()
    english_word = data.get("english_word")
    russian_word = data.get("russian_word")
    user_id = message.from_user.id

    try:
        ########################################################################
        #                        DATABASE Queries                              #
        current_dictionary = await db.select_current_dictionary(user_id)
        await db.add_translate(english_word, russian_word, current_dictionary)
        translate = await db.select_last_translate(current_dictionary)
        translate_id = translate[0]
        await db.set_learning_translate(current_dictionary, translate_id)
        ########################################################################
        await message.answer(f"Добавлен новый перевод:\n"
                             f"{english_word} - {russian_word}", reply_markup=menu)
        await state.finish()

    except asyncpg.exceptions.NotNullViolationError:
        select_dictionaries = await db.select_dictionaries(user_id)
        if len(select_dictionaries) > 0:
            """It raised when user delete current dictionary and try to add new translate"""
            show_dictionaries_keyboard = InlineKeyboardMarkup(
                inline_keyboard=[
                    [InlineKeyboardButton(
                        text=item[2],
                        callback_data=select_dictionary_callback.new(
                            dictionary_id=item[0],
                            dictionary_name=item[2]))] for item in select_dictionaries]
            )  # Generation dictionaries into inline buttons
            await message.answer("Так как вы удалили свой текущий словарь, \n"
                                 "то вам придется выбрать новый текущий словарь",
                                 reply_markup=show_dictionaries_keyboard)  # Showing all user dictionaries for select
        else:
            """It raised when user delete all dictionaries and try to add new translate"""
            await message.answer("У вас нету словарей! Добавьте хоть один словарь.", reply_markup=menu)
            await state.finish()
    except:
        await message.answer("Упс, какая-то ошибка!\n ", reply_markup=menu)
        await state.finish()

@dp.callback_query_handler(select_dictionary_callback.filter(), state=CreateNewTranslate.SetRussianWord)
async def changing_dictionary(call: CallbackQuery, callback_data: dict, state: FSMContext):
    """Setting current dictionary from exception"""
    await call.message.delete()
    name_selected_dictionary = callback_data['dictionary_name']
    user_id = call.from_user.id
    id_selected_dictionary = int(callback_data['dictionary_id'])
    ########################################################################
    #                        DATABASE Queries                              #
    await db.set_current_dictionary(user_id, id_selected_dictionary)
    ########################################################################
    await call.answer(f"Выбран словарь '{name_selected_dictionary}'", cache_time=5)
    await call.message.answer("Теперь вы можете создать перевод!", reply_markup=menu)
    await state.finish()


import cv2
import glob
import numpy as np
import math
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib.mlab as mlab
from scipy import stats

img_path_pos = glob.glob("D:/winuser/Desktop/HW3/dataset/pos/*.jpg")
img_path_neg = glob.glob("D:/winuser/Desktop/HW3/dataset/neg/*.jpg")
pos_mean_matrix = np.zeros((10, 10), dtype=np.float)
neg_mean_matrix = np.zeros((10, 10), dtype=np.float)

for path1 in img_path_pos:
    img_pos = cv2.imread(path1)
    img_pos_resize = cv2.resize(img_pos, (10, 10))
    img_pos_gray = cv2.cvtColor(img_pos_resize, cv2.COLOR_BGR2GRAY)
    pos_mean_matrix += img_pos_gray / len(img_path_pos)
cv2.imwrite("D:/winuser/Desktop/DM and Vis/HW3/pos_mean_matrix.jpg", pos_mean_matrix)
for path1 in img_path_neg:
    img_neg = cv2.imread(path1)
    img_neg_resize = cv2.resize(img_neg, (10, 10))
    img_neg_gray = cv2.cvtColor(img_neg_resize, cv2.COLOR_BGR2GRAY)
    neg_mean_matrix += img_neg_gray / len(img_path_neg)

plt.hist(pos_mean_matrix.flatten(), normed=True, color='pink', edgecolor='k')
plt.hist(neg_mean_matrix.flatten(), normed=True, color='lightblue', edgecolor='k')

line1 = sns.kdeplot(pos_mean_matrix.flatten(), color='r', linewidth=2, label='pos')
line2 = sns.kdeplot(neg_mean_matrix.flatten(), color='b', linewidth=2, label='neg')
plt.tick_params(top='off', right='off')
plt.xlabel('Brightness distribution(0 ~ 255)')
plt.ylabel('Probability')
plt.legend([line1, line2], labels=['pos', 'neg'], loc='best')
plt.savefig('D:/winuser/Desktop/DM and Vis/HW3/pic.jpg')
plt.close('all')

gaussian1 = stats.norm(loc=pos_mean_matrix.mean(), scale=1.0)
gaussian2 = stats.norm(loc=neg_mean_matrix.mean(), scale=1.0)
x2 = np.linspace(113.0, 140.0, 1000)
y1 = gaussian1.pdf(x2)
y2 = gaussian2.pdf(x2)
plt.xlabel('Brightness distribution(0 ~ 255)')
plt.ylabel('Probability')
a1 = plt.plot(x2, y1, color='red', label="pos")
a2 = plt.plot(x2, y2, color='blue', label='neg')
plt.legend([a1, a2], labels=['pos', 'neg'], loc='best')
plt.savefig('D:/winuser/Desktop/DM and Vis/HW3/gaussian.jpg')

for k in range(2):
    test = cv2.imread("D:/winuser/Desktop/DM and Vis/HW3/test{}.jpg".format(k + 1))
    test_gray = cv2.cvtColor(test, cv2.COLOR_BGR2GRAY)
    test_h = test_gray.shape[0]
    test_w = test_gray.shape[1]
    for i in range(math.floor(test_h / 10) - 1):
        for j in range(math.floor(test_w / 10) - 1):
            crop_test = test_gray[i * 10: (i + 1) * 10, j * 10: (j + 1) * 10]
            pixel = 0
            for m in range(10):
                for n in range(10):
                    pixel += crop_test[m, n]
            pixel_mean = pixel / 100
            if pixel_mean >= (pos_mean_matrix.mean() + neg_mean_matrix.mean()) / 2:
                cv2.rectangle(test, (j * 10, i * 10), ((j + 1) * 10, (i + 1) * 10), (0, 255, 0), 2)
        cv2.imwrite("D:/winuser/Desktop/DM and Vis/HW3/test{}_result1.jpg".format(k + 1), test)


import sqlalchemy as sa
from sqlalchemy import BOOLEAN, FunctionElement, TypeDecorator
from sqlalchemy.dialects import postgresql, sqlite
from sqlalchemy.ext.compiler import compiles

from ..util import json_dumps

class json_contains(FunctionElement):
    """
    Platform independent json_contains operator, tests if the
    `left` expression contains the `right` expression.

    On postgres this is equivalent to the @> containment operator.
    https://www.postgresql.org/docs/current/functions-json.html
    """

    type = BOOLEAN
    name = "json_contains"
    inherit_cache = False

    def __init__(self, left, right):
        self.left = left
        self.right = right
        super().__init__()

@compiles(json_contains, "postgresql")
def _json_contains_postgresql(element, compiler, **kwargs):
    return compiler.process(
        sa.type_coerce(element.left, postgresql.JSONB).contains(
            sa.type_coerce(element.right, postgresql.JSONB)
        ),
        **kwargs,
    )

def _json_contains_sqlite_fn(left, right, compiler, **kwargs):
    if isinstance(left, (list, dict, tuple, str)):
        left = json_dumps(left)

    if isinstance(right, (list, dict, tuple, str)):
        right = json_dumps(right)

    json_each_left = sa.func.json_each(left).alias("left")
    json_each_right = sa.func.json_each(right).alias("right")

    distinct_matches = (
        sa.select(sa.func.count(sa.distinct(sa.literal_column("left.value"))))
        .select_from(json_each_left)
        .join(
            json_each_right,
            sa.literal_column("left.value") == sa.literal_column("right.value"),
        )
        .scalar_subquery()
    )

    distinct_keys = (
        sa.select(sa.func.count(sa.distinct(sa.literal_column("right.value"))))
        .select_from(json_each_right)
        .scalar_subquery()
    )

    return compiler.process(distinct_matches >= distinct_keys)

@compiles(json_contains, "sqlite")
def _json_contains_sqlite(element, compiler, **kwargs):
    return _json_contains_sqlite_fn(element.left, element.right, compiler, **kwargs)

class json_has_any_key(FunctionElement):
    """
    Platform independent json_has_any_key operator.

    On postgres this is equivalent to the ?| existence operator.
    https://www.postgresql.org/docs/current/functions-json.html
    """

    type = BOOLEAN  # type: ignore
    name = "json_has_any_key"
    inherit_cache = False

    def __init__(self, json_expr, values: list):
        self.json_expr = json_expr
        if not all(isinstance(v, str) for v in values):
            raise ValueError("json_has_any_key values must be strings")
        self.values = values
        super().__init__()

@compiles(json_has_any_key, "postgresql")
@compiles(json_has_any_key)
def _json_has_any_key_postgresql(element, compiler, **kwargs):

    values_array = postgresql.array(element.values)
    # if the array is empty, postgres requires a type annotation
    if not element.values:
        values_array = sa.cast(values_array, postgresql.ARRAY(sa.String))

    return compiler.process(
        sa.type_coerce(element.json_expr, postgresql.JSONB).has_any(values_array),
        **kwargs,
    )

@compiles(json_has_any_key, "sqlite")
def _json_has_any_key_sqlite(element, compiler, **kwargs):
    json_each = sa.func.json_each(element.json_expr).alias("json_each")
    return compiler.process(
        sa.select(1)
        .select_from(json_each)
        .where(
            sa.literal_column("json_each.value").in_(
                sa.bindparam(key="json_each_values", value=element.values, unique=True)
            )
        )
        .exists(),
        **kwargs,
    )

class json_has_all_keys(FunctionElement):
    """Platform independent json_has_all_keys operator.

    On postgres this is equivalent to the ?& existence operator.
    https://www.postgresql.org/docs/current/functions-json.html
    """

    type = BOOLEAN  # type: ignore
    name = "json_has_all_keys"
    inherit_cache = False

    def __init__(self, json_expr, values: list):
        self.json_expr = json_expr
        if isinstance(values, list) and not all(isinstance(v, str) for v in values):
            raise ValueError(
                "json_has_all_key values must be strings if provided as a literal list"
            )
        self.values = values
        super().__init__()

@compiles(json_has_all_keys, "postgresql")
@compiles(json_has_all_keys)
def _json_has_all_keys_postgresql(element, compiler, **kwargs):
    values_array = postgresql.array(element.values)

    # if the array is empty, postgres requires a type annotation
    if not element.values:
        values_array = sa.cast(values_array, postgresql.ARRAY(sa.String))

    return compiler.process(
        sa.type_coerce(element.json_expr, postgresql.JSONB).has_all(values_array),
        **kwargs,
    )

@compiles(json_has_all_keys, "sqlite")
def _json_has_all_keys_sqlite(element, compiler, **kwargs):
    # "has all keys" is equivalent to "json contains"
    return _json_contains_sqlite_fn(
        left=element.json_expr,
        right=element.values,
        compiler=compiler,
        **kwargs,
    )

class JSON(TypeDecorator):
    """
    JSON type that returns SQLAlchemy's dialect-specific JSON types, where
    possible. Uses generic JSON otherwise.

    The "base" type is postgresql.JSONB to expose useful methods prior
    to SQL compilation
    """

    impl = postgresql.JSONB
    cache_ok = True

    def load_dialect_impl(self, dialect):
        if dialect.name == "postgresql":
            return dialect.type_descriptor(postgresql.JSONB(none_as_null=True))
        elif dialect.name == "sqlite":
            return dialect.type_descriptor(sqlite.JSON(none_as_null=True))
        else:
            return dialect.type_descriptor(sa.JSON(none_as_null=True))

    class comparator_factory(sa.JSON.Comparator):
        def contains(self, other, **kw):
            json_contains(self, other)

        def has_any_key(self, other):
            return json_has_any_key(self, other)

        def has_all_keys(self, other):
            return json_has_all_keys(self, other)


from distutils.core import setup
from setuptools import find_packages

install_requires = [
    'decorator<5.0.0',  # For networkx
    'networkx>=2.5',
    'numpy>1.7.0',
    'Pillow>=8.3.2',
    'pynput>=1.7.3',
    'pyzmq>=20.0.0',
    'av>=8.0.1',
    'sounddevice>=0.4.3',
    'soundfile>=0.10.3',
    'msgpack>1.0.0',
    'windows-capture-devices; platform_system == "Windows"'
]

setup(
    name='multisensor-pipeline',
    version='2.1.1',
    author='Michael Barz',
    author_email='michael.barz@dfki.de',
    license='CC BY-NC-SA 4.0',
    packages=find_packages(
        include=('multisensor_pipeline.*', 'multisensor_pipeline')
    ),
    url="https://github.com/DFKI-Interactive-Machine-Learning/multisensor-pipeline",
    description="The core library of the DFKI multisensor pipeline framework.",
    python_requires='>=3.6.0',
    install_requires=install_requires,
    keywords=[
        'multimodality', 'streaming', 'multisensor', 'sensors', 'multimodal interaction',
        'pipeline', 'stream processing', 'multiprocessing'
    ]
)


import configparser
import os
import json
import urllib.request as urlRequest
from bs4 import BeautifulSoup
import time
import datetime
import traceback
from threading import Thread as thread
import threading
class aqi(thread):
    def __init__(self,file,conn):
        thread.__init__(self)
        self.__flag = threading.Event()
        self.__flag.set()
        self.__running = threading.Event()
        self.__running.set()

        self.configFile = os.getcwd() + file
        self.conn = conn
        self.exception = None

    def pause(self):
        self.__flag.clear()
    def resume(self):
        self.__flag.set()
    def stop(self):
        self.__flag.set()
        self.__running.clear()
    def spider(self):
        self.conn.saveLogInfo("aqi", "正在抓取AQI空气信息....",
                                  datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), 1)
        print(datetime.datetime.now().strftime(
            "%Y-%m-%d %H:%M:%S") + " aqi " + "正在抓取AQI空气信息...  " + "运行状态: " + str(1))
        url = self.url + self.token
        req = urlRequest.Request(url)
        res = urlRequest.urlopen(req)
        if res != None:
            soup = BeautifulSoup(res,"lxml")
            aqiDetailJson = json.loads(soup.text)
            ## 使用json 文件测试
            # jsonFile = os.getcwd()+"/json/ariQuality.json"
            # fp = open(jsonFile,encoding="utf-8")
            # aqiDetailJson = json.load(fp)
            if "error" in aqiDetailJson:
                self.conn.saveLogInfo("aqi", aqiDetailJson["error"], datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), 1)
                print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + " aqi " + aqiDetailJson["error"] +"  " + "运行状态: " + str(1))
                return -1
            for aqi in aqiDetailJson:
                if aqi["position_name"] == None:
                    aqiDetailJson.remove(aqi)
            self.conn.saveAqiData(aqiDetailJson,self.dbNameList)

        # tabelContent = soup.find("table", {"id": self.tabelId})
        # timeClass = soup.find("div",{"class":"live_data_time"})
        # time = re.findall("时间：(.*)",timeClass.find("p").text)[0]
        # tb = tabelContent.find("tbody")

        # if tb:
        #     for tr in tb.findAll("tr"):
        #
        #         order = 1
        #         resultDict = dict()
        #         for td in tr.findAll("td"):
        #             if "class" in td.attrs:
        #                 continue
        #             resultDict[self.tdOrder[order]] = td.text
        #             order += 1
        #         resultDict["timepoint"]=time
        #         resultDict["co_24h"]= '0.7'
        #         resultDict['no_24h'] = "0.591"
        #         resultDict['o3_24h'] = ''
        #         resultDict['o3_24h'] =47
        #         resultDict['pm10_24h'] = 35
        #         resultDict['pm25_24h'] = 28
        #         ## TODO 插入数据库
        #         print("插入数据库")
        #         sqlStatment = "INSERT INTO media.airquality ('positionname','timepoint','co','co_24h','no','no_24h'," \
        #                       "'o3','o3_24h','pm10','pm10_24h','pm25','pm25_24h','so2','so2_24h','primarypollutant','quality')" \
        #                       "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" \
        #                       %(resultDict['positionname'],resultDict['timepoint'],resultDict['co'],resultDict['co_24h'],resultDict['no'],
        #                         resultDict['no_24h'],resultDict['o3'],resultDict['o3_24h'],resultDict['pm10'],resultDict['pm10_24h'],resultDict['pm25'],
        #                         resultDict['pm25_24h'],resultDict['so2'],resultDict['so2_24h'],resultDict['primarypollutant'],resultDict['quality'])
        #         self.conn.sqlExec(sqlStatment,type=1)

    def loadConf(self):
        self.conn.saveLogInfo("aqi", "读取AQI配置文件", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), 1)
        print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + " aqi " + "读取AQI配置文件  " + "运行状态: " + str(1))
        conf = configparser.ConfigParser()
        conf.read(self.configFile,"utf-8")
        self.url = conf.get(conf.sections()[0],"url")

        self.dbNameList = conf.get(conf.sections()[1],"dbDict").split(",")

        self.token = conf.get(conf.sections()[1],"token")
    ## Aqi数据每半小时抓取一次
    def run(self):
        self.conn.saveLogInfo("aqi", "AQI线程开始运行",datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), 1)
        print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + " aqi " + "AQI线程开始运行  " + "运行状态: " + str(1))
        try:
            while self.__running.isSet():
                self.__flag.wait()
                self.conn.saveLogInfo("aqi", "AQI信息开始抓取", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), 1)
                print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + " aqi " + "AQI信息开始抓取  " + "运行状态: " + str(1))
                self.spider()
                self.conn.saveLogInfo("aqi", "AQI信息抓取完毕", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), 1)
                print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + " aqi " + "AQI信息抓取完毕  " + "运行状态: " + str(1))
                time.sleep(2700)
        except Exception as e:
            self.exTrace = traceback.format_exc()
            self.exception = e
            traceback.print_exc()
    def getException(self):
        if self.exception:
            exString = ""
            for arg in self.exception.args:
                exString += str(arg)
            if not self.exTrace:
                exString += self.exTrace
            exString = exString.replace("'", "")
            self.conn.saveLogInfo("aqi", "AQI线程异常,异常内容为%s"%exString, datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), 0)
            print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + " aqi " + "AQI线程异常,异常内容为%s  "%exString + "运行状态: " + str(0))
        else:
            self.conn.saveLogInfo("aqi", "aqi线程异常内容为空", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), 0)
            print(datetime.datetime.now().strftime( "%Y-%m-%d %H:%M:%S") + " aqi " + "AQI线程异常,异常内容为空  " + "运行状态: " + str(0))
    def dbReset(self,db):
        self.conn = db


# -*- coding: utf-8 -*-

import cupy     as cp

from .util import (
    judge_dtype,
    elementwise_not,
    ternary_operation,
    checkfinite,
)

#############################
#   fixpix
#############################

def fixpix(data,mask,out=None,dtype=None,fix_NaN=False):
    '''
    fill the bad pixel with linear interpolation using surrounding pixels

    Parameters
    ----------
    data : array-like
        An array of image
        If a 3D array containing multiple images,
        the images must be stacked along the 1st dimension (axis=0).
    mask : array-like
        An array indicates bad pixel positions
        The shape must be same as image.
        The value of bad pixel is nonzero, and the others is 0.
        If all pixels are bad, raise ValueError.
    out : cupy.ndarray, default None
        Alternate output array in which to place the result. The default
        is None; if provided, it must have the same shape as the
        expected output, but the type will be cast if necessary.
    dtype : str or dtype, default None
        dtype of ndarray used internally
        If None, use eclair.common.default_dtype.
        If the input dtype is different, use a casted copy.
    fix_NaN : bool, default False
        If true, fix NaN pixel even if it's not specified
        as bad pixel in mask

    Returns
    -------
    fixed : ndarray
        An array of images fixed bad pixel

    Notes
    -----
    NaN is ignored in interpolation calculations,
    but is not fixed if fix_NaN is False.
    '''
    dtype = judge_dtype(dtype)

    data = cp.asarray(data,dtype=dtype)
    mask = cp.asarray(mask,dtype=dtype)

    dshape = data.shape
    imshape = data.shape[-2:]
    if imshape != mask.shape[-2:]:
        raise ValueError('shape differs between data and mask')
    if mask.all(axis=(-2,-1)).any():
        raise ValueError('No available pixel')

    if out is None:
        out = data.copy()
    else:
        cp.copyto(out,data)

    tout = cp.array(out,copy=False,ndmin=3)
    filt = cp.array(mask,ndmin=3)
    elementwise_not(filt,filt)

    ternary_operation(filt,tout,0,tout)

    dconv = cp.empty_like(tout)
    nconv = cp.empty_like(filt)

    for _ in range(max(imshape)):
        conv_kernel(tout,dconv)
        conv_kernel(filt,nconv)
        fix_core(filt,dconv,nconv,fix_NaN,tout)
        if nconv.all():
            break
        else:
            cp.sign(nconv,out=filt)

    return out

fix_core = cp.ElementwiseKernel(
    in_params='F f, T d, F n, bool fn',
    out_params='T z',
    operation='''
        int flag = (!fn | isfinite(z)) & (f!=0);
        T val = d / (n+(n==0));
        z = (flag ? z : val);
    ''',
    name='fix_core'
)

conv_kernel = cp.ElementwiseKernel(
    in_params='raw T input',
    out_params='T output',
    operation='''
        int ly = input.shape()[1];
        int lx = input.shape()[2];
        int iyz = i / lx;
        int i_x = i % lx;
        int i_y = iyz % ly;
        int i_z = iyz / ly;
        int idx[3] = {i_z};
        int *t_y = &(idx[1]), *t_x = &(idx[2]);
        int s_x = max(i_x-1,0), e_x = min(i_x+2,lx);
        int s_y = max(i_y-1,0), e_y = min(i_y+2,ly);
        int fn, c = 0;
        T val, tmp, summed = 0;
        for (*t_y=s_y; *t_y<e_y; (*t_y)++) {
            tmp = 0;
            for (*t_x=s_x; *t_x<e_x; (*t_x)++) {
                val = input[idx];
                fn = isfinite(val);
                tmp += (fn ? val : 0);
                c += fn;
            }
            summed += tmp;
        }
        output = summed / (c + (c==0))
    ''',
    name='convolution'
)


from pyspark.sql import SparkSession
from pyspark.sql.functions import *

import sys

spark = SparkSession.builder.appName("A3T1").getOrCreate()

# Retrive word and the value of K
wrd = sys.argv[1]
k = int(sys.argv[2])

# Reading the two csv files
shape = spark.read.option("header",True).csv(sys.argv[3])
shape_stat = spark.read.option("header",True).csv(sys.argv[4])

# Joining the Dataframes
joined = shape_stat.join(shape, ['key_id','word'])

# Filters

# Check presence of word
joined = joined.filter(joined.word.contains(wrd))

# Check recog status = False
joined = joined.filter(joined.recognized.contains("False"))

# Check num strokes < K
joined = joined.filter(joined.Total_Strokes < k)

# Grouping Entries by country and sorting them
final = joined.groupBy("countrycode").count()
final = final.orderBy("countrycode")

if(len(final.collect()) > 0):
	for i in final.collect():
		print(i[0], i[1], sep=",")
else:
	print("0")


from django.shortcuts import render
from django.http import HttpResponseRedirect
from django.contrib.auth.decorators import login_required
from django.contrib import messages
from django.core.paginator import Paginator
from django.db.models import Q

from .forms import *
from .models import *

from constel.apps.controle_acessos.decorator import permission
from ...menu import menu_principal, menu_cadastros, menu_entradas, menu_saidas, menu_consultas, menu_consultas_modelos

from constel.forms import FormFiltraQ

@login_required
@permission('patrimonio', )
def cadastra_ferramenta(request):
    menu = menu_cadastros(request)

    if request.method == 'POST':
        form = FormCadastraFerramenta(request.POST)

        if form.is_valid():
            ferramenta = Ferramenta(
                nome=form.cleaned_data['nome'],
                descricao=form.cleaned_data['descricao'],
                user=request.user,
            )
            ferramenta.save()
            FerramentaQuantidade(
                ferramenta=ferramenta,
                quantidade=0,
            ).save()

            return HttpResponseRedirect('/patrimonio/cadastros/ferramenta')
    else:
        form = FormCadastraFerramenta()

    itens = Ferramenta.objects.values(
        'nome',
        'data',
        'user__first_name',
        'user__last_name',
    ).order_by('-data')

    paginator = Paginator(itens, 50)
    page_number = request.GET.get('page')
    page_obj = paginator.get_page(page_number)

    context = {
        'page_obj': page_obj,
        'form': form,
        'form_submit_text': 'Cadastrar nova ferramenta',
    }
    context.update(menu)

    return render(request, 'ferramenta/v2/cadastra_ferramenta.html', context)

@login_required
@permission('patrimonio', )
def entrada_ferramenta(request):
    menu = menu_entradas(request)

    if request.method == 'POST':
        form = FormEntradaFerramenta(request.POST)

        if form.is_valid():
            FerramentaEntrada(
                ferramenta=form.cleaned_data['ferramenta'],
                quantidade=form.cleaned_data['quantidade'],
                valor=form.cleaned_data['valor'],
                observacao=form.cleaned_data['observacao'],
                user=request.user,
            ).save()
            ferramenta = form.cleaned_data['ferramenta']
            ferramenta.quantidade.quantidade += form.cleaned_data['quantidade']
            ferramenta.quantidade.save()

            return HttpResponseRedirect('/patrimonio/entradas/ferramenta')

    else:
        form = FormEntradaFerramenta()

    context = {
        'form': form,
        'form_submit_text': 'Registrar entrada',
    }
    context.update(menu)

    return render(request, 'patrimonio/v2/entrada.html', context)

@login_required
@permission('patrimonio', )
def saida_ferramenta(request):
    menu = menu_saidas(request)

    if request.method == 'POST':
        form = FormSaidaFerramenta(request.POST)

        if form.is_valid():
            saida = FerramentaSaida(
                ferramenta=form.cleaned_data['ferramenta'],
                quantidade=form.cleaned_data['quantidade'],
                observacao=form.cleaned_data['observacao'],
                user_to=form.cleaned_data['user_to'],
                user=request.user,
            )
            ferramenta = form.cleaned_data['ferramenta']
            ferramenta.quantidade.quantidade -= form.cleaned_data['quantidade']

            if FerramentaQuantidadeFuncionario.objects.filter(
                user=form.cleaned_data['user_to'],
                ferramenta=ferramenta
            ).exists():

                carga = FerramentaQuantidadeFuncionario.objects.get(
                    user=form.cleaned_data['user_to'],
                    ferramenta=ferramenta
                )
                carga.quantidade += form.cleaned_data['quantidade']

            else:
                carga = FerramentaQuantidadeFuncionario(
                    user=form.cleaned_data['user_to'],
                    ferramenta=ferramenta,
                    quantidade=form.cleaned_data['quantidade']
                )

            ferramenta.quantidade.save()
            saida.save()
            carga.save()

            return HttpResponseRedirect('/patrimonio/saidas/ferramenta')

    else:
        form = FormSaidaFerramenta()

    context = {
        'form': form,
        'form_submit_text': 'Registrar entrega',
    }
    context.update(menu)

    return render(request, 'patrimonio/v2/entrada.html', context)

@login_required
@permission('patrimonio', )
def fechamento_ferramenta(request):
    menu = menu_principal(request)

    if request.method == 'POST':
        form = FormFechamentoFerramenta(request.POST)

        if form.is_valid():
            ferramenta = form.cleaned_data['ferramenta']
            status = form.cleaned_data['status']

            fechamento = FerramentaFechamento(
                ferramenta=ferramenta,
                status=form.cleaned_data['status'],
                quantidade=form.cleaned_data['quantidade'],
                observacao=form.cleaned_data['observacao'],
                user_from=form.cleaned_data['user_from'],
                user=request.user,
            )

            if int(status) == 0:
                ferramenta.quantidade.quantidade += form.cleaned_data['quantidade']
                messages.success(request, 'Fechamento realizado e ferramentas (RE)inseridas no estoque')

            else:
                messages.success(request, 'Fechamento realizado')

            carga = FerramentaQuantidadeFuncionario.objects.get(
                user=form.cleaned_data['user_from'],
                ferramenta=ferramenta
            )
            carga.quantidade -= form.cleaned_data['quantidade']

            fechamento.save()
            ferramenta.quantidade.save()
            carga.save()

            return HttpResponseRedirect('/patrimonio/fechamento/ferramenta')

    else:
        form = FormFechamentoFerramenta()

    context = {
        'form': form,
        'form_submit_text': 'Registrar fechamento',
    }
    context.update(menu)

    return render(request, 'patrimonio/v2/entrada.html', context)

@login_required
@permission('patrimonio', )
def consulta_ferramenta(request):
    menu = menu_consultas_modelos(request)

    ferramenta = request.GET.get('q', '')

    form = FormFiltraQ(
        descricao="ferramenta",
        initial={
            'q': ferramenta,
        }
    )

    query = Q()

    if ferramenta != '':
        query = query & Q(nome__icontains=ferramenta)

    itens = Ferramenta.objects.filter(
        query
    ).values(
        'id',
        'nome',
        'descricao',
        'data',
        'user__first_name',
        'user__last_name',
    ).order_by('nome')

    paginator = Paginator(itens, 50)
    page_number = request.GET.get('page')
    page_obj = paginator.get_page(page_number)

    context = {
        'form': form,
        'form_submit_text': 'Filtrar',
        'page_obj': page_obj,
    }
    context.update(menu)

    return render(request, 'ferramenta/v2/consulta_ferramenta.html', context=context)

def edita_modelo_ferramenta(request, modelo_id):
    menu = menu_consultas_modelos(request)

    modelo = Ferramenta.objects.get(id=modelo_id)

    form = FormEditaModeloFerramenta(data=request.POST or None, instance=modelo)

    if request.method == 'POST':

        if form.is_valid():
            form.save()

            messages.success(request, 'Alterações salvas com sucesso')

            return HttpResponseRedirect(f'/patrimonio/edicao/ferramenta/modelo/{modelo_id}')

    context = {
        'form': form,
        'form_submit_text': 'Salvar alterações',
    }
    context.update(menu)

    return render(request, 'ferramenta/v2/edita_modelo_ferramenta.html', context)

@login_required
@permission('patrimonio', )
def consulta_ferramenta_estoque(request):
    menu = menu_consultas(request)

    ferramenta = request.GET.get('q', '')

    form = FormFiltraQ(
        descricao="ferramenta",
        initial={
            'q': ferramenta,
        }
    )

    query = Q()

    if ferramenta != '':
        query = query & Q(ferramenta__nome__icontains=ferramenta)

    itens = FerramentaQuantidade.objects.filter(
        query
    ).values(
        'ferramenta__nome',
        'quantidade',
    ).exclude(
        quantidade__lte=0,
    ).order_by('ferramenta__nome')

    paginator = Paginator(itens, 50)
    page_number = request.GET.get('page')
    page_obj = paginator.get_page(page_number)

    context = {
        'form': form,
        'form_submit_text': 'Filtrar',
        'page_obj': page_obj,
    }
    context.update(menu)

    return render(request, 'ferramenta/v2/consulta_ferramenta_estoque.html', context=context)


from random import randint

import pygame

pygame.init()
W = 300
H = 300
WHITE = (255, 255, 255)

class Car(pygame.sprite.Sprite):
    def __init__(self, x, filename):
        pygame.sprite.Sprite.__init__(self)
        self.image = pygame.image.load(filename).convert_alpha()
        self.rect = self.image.get_rect(center=(x, 0))

sc = pygame.display.set_mode((W, H))
# координата x будет случайна
car1 = Car(randint(1, W), 'data/hero.png')
car1.rect.x, car1.rect.y = 100, 100
while 1:
    for i in pygame.event.get():
        if i.type == pygame.QUIT:
            exit()
        elif i.type == pygame.KEYDOWN:
            if i.key == pygame.K_UP:
                car1.rect.y -= 10
            if i.key == pygame.K_DOWN:
                car1.rect.y += 10
            if i.key == pygame.K_RIGHT:
                car1.rect.x += 10
            if i.key == pygame.K_LEFT:
                car1.rect.x -= 10
    sc.fill(WHITE)
    sc.blit(car1.image, car1.rect)
    pygame.display.update()


import torch
from nuclr.data import prepare_nuclear_data
from nuclr.config import NUCLR
import matplotlib.pyplot as plt
from nuclr.model import get_model_and_optim
from lib.config_utils import parse_arguments_and_get_name
import os

args, name = parse_arguments_and_get_name(NUCLR)
torch.manual_seed(args.SEED)

data = prepare_nuclear_data(args)

print(args)

def load(path):
    return torch.load(path, map_location=torch.device('cpu'))

basedir = os.path.join(args.ROOT, name)
modelpath = os.path.join(basedir, "model_NUCLR_best.pt")
model, _ = get_model_and_optim(data, args)
model.load_state_dict(load(modelpath))
model.eval()

DEVICE = args.DEV

def get_idx(data, target):
    output_map = data.output_map
    if target not in output_map:
        raise ValueError(f"Target {target} not in output map")
    target_size = output_map[target]
    if target_size > 1:
        raise ValueError(f"Target {target} is not a scalar")
    target_idx = 0
    for prev_target, size in output_map.items():
        if prev_target == target:
            return target_idx
        target_idx += size

def plot(target="binding_energy", split="both", rescale=False):
    true = data.y
    with torch.no_grad(): pred = model(data.X)
    if rescale:
        true = torch.tensor(data.regression_transformer.inverse_transform(true))
        pred = torch.tensor(data.regression_transformer.inverse_transform(pred))
    target_idx = get_idx(data, target)
    target_true = true[:, target_idx]
    target_pred = pred[:, target_idx]
    if split == "train":
        split_mask = data.train_mask
    elif split == "val":
        split_mask = data.val_mask
    elif split == "both":
        split_mask = data.train_mask | data.val_mask
    else:
        raise ValueError(f"Unknown split {split}")
    nanmask = ~target_true.isnan()
    mask = split_mask & nanmask

    _, axes = plt.subplots(2, 2, figsize=(20, 14))
    axes = axes.flatten()
    for ax, target in zip(axes, [target_true, target_true - target_pred]):
        sc = ax.scatter(data.X[:, 0][mask], data.X[:, 1][mask], c=target[mask], s=10, cmap="bwr")
        ax.set_xlabel("Z")
        ax.set_ylabel("N")
        #center the colorbar around 0
        if ax == axes[1]:
          max_ = max(abs(sc.get_clim()[0]), abs(sc.get_clim()[1]))
          sc.set_clim(-max_, max_)
        plt.colorbar(sc, ax=ax)
    for ax, title in zip(axes, ["True", "True - Pred"]):
        ax.set_title(title)
    for ax, x in zip(axes[2:], data.X.T):
        ax.scatter(x, target_true, s=10, label="True")
        ax.scatter(x, target_pred, s=10, label="Pred")
        ax.set_xlabel("N" if ax == axes[2] else "Z")
        ax.set_ylabel("Binding energy")
        ax.legend()
    plt.tight_layout()
    plt.show()

if __name__ == "__main__":
    plot("binding_energy", split="val", rescale=True)


#! /usr/bin/env python

from __future__ import print_function

import numpy as np
import pandas as pd

import ROOT
# disable cmd line parsing before other ROOT deps are loaded
ROOT.PyConfig.IgnoreCommandLineOptions = True
from root_pandas import read_root

import matplotlib.pyplot as plt

from sklearn.cross_validation import KFold, train_test_split
from sklearn.grid_search import RandomizedSearchCV
from xgboost import XGBClassifier
from scripts.metrics import tagging_power_score
from uncertainties import ufloat

from scripts.calibration import PolynomialLogisticRegression
from scripts.data_preparation import NSplit

from collections import OrderedDict
from itertools import islice, product
from tqdm import tqdm
import argparse
import datetime
import json
from os import path, mkdir

def get_event_number(df, weight_column='SigYield_sw'):
    """ Use weighted sums
    """
    # max, min, mean, first should give the same values here
    return np.sum(df.groupby('event_id')[weight_column].first())

class CutBasedXGBClassifier(XGBClassifier):
    def __init__(self, max_depth=3, learning_rate=0.1, n_estimators=100,
                 silent=True, objective="reg:linear",
                 nthread=1, gamma=0, min_child_weight=1, max_delta_step=0,
                 subsample=1, colsample_bytree=1, colsample_bylevel=1,
                 reg_alpha=0, reg_lambda=1, scale_pos_weight=1,
                 base_score=0.5, seed=0, missing=None,
                 P_column='tp_partP', P_cut=0,
                 PT_column='tp_partPt', PT_cut=1.1,
                 phiDistance_column='tp_minPhiDistance', phiDistance_cut=0.005,
                 MuonPIDIsMuon_column='tp_MuonPIDIsMuon', MuonPIDIsMuon_cut=1,
                 mvaFeatures=None, only_max_pt=True,
                 event_identifier_column='event_id',
                 IsSignalDaughter_column='tp_IsSignalDaughter',
                 IsSignalDaughter_cut=0,
                 RecVertexIPs_column='tp_ABS_RecVertexIP',
                 RecVertexIPs_cut=0,
                 TRCHI2DOF_column='tp_partlcs', TRCHI2DOF_cut=3,
                 TRGHP_column='tp_ghostProb', TRGHP_cut=0.4,
                 PROBNNmu_column='tp_PIDNNm', PROBNNmu_cut=0.35,
                 PROBNNpi_column='tp_PROBNNpi', PROBNNpi_cut=0.8,
                 PROBNNe_column='tp_PROBNNe', PROBNNe_cut=0.8,
                 PROBNNk_column='tp_PROBNNk', PROBNNk_cut=0.8,
                 PROBNNp_column='tp_PROBNNp', PROBNNp_cut=0.8,
                 IPPUs_column='tp_IPPU', IPPUs_cut=3,
                 polynomial_pow=3,
                 ):
        self.cut_parameters = ['P', 'PT', 'phiDistance', 'MuonPIDIsMuon',
                               'IsSignalDaughter', 'TRCHI2DOF', 'TRGHP',
                               'PROBNNmu', 'PROBNNpi', 'PROBNNe', 'PROBNNk',
                               'PROBNNp', 'IPPUs', 'RecVertexIPs',
                               ]
        for cp in self.cut_parameters:
            setattr(self,
                    '{}_cut'.format(cp),
                    locals()['{}_cut'.format(cp)])
            setattr(self,
                    '{}_column'.format(cp),
                    locals()['{}_column'.format(cp)])
        self.calibrator = PolynomialLogisticRegression(power=polynomial_pow,
                                                       solver='lbfgs',
                                                       n_jobs=nthread)
        self.mvaFeatures = mvaFeatures
        self.only_max_pt = only_max_pt
        self.event_identifier_column = event_identifier_column
        self.fit_successful = False
        (super(CutBasedXGBClassifier, self)
         .__init__(max_depth=max_depth, learning_rate=learning_rate,
                   n_estimators=n_estimators,
                   silent=silent, objective=objective,
                   nthread=nthread, gamma=gamma,
                   min_child_weight=min_child_weight,
                   max_delta_step=max_delta_step,
                   subsample=subsample, colsample_bytree=colsample_bytree,
                   colsample_bylevel=colsample_bylevel,
                   reg_alpha=reg_alpha, reg_lambda=reg_lambda,
                   scale_pos_weight=scale_pos_weight,
                   base_score=base_score, seed=seed, missing=None))

    def select(self, X, y=None):
        len_before = get_event_number(X)
        selection = ((X[self.P_column] > self.P_cut) &
                     (X[self.PT_column] > self.PT_cut) &
                     (X[self.phiDistance_column] > self.phiDistance_cut) &
                     (X[self.MuonPIDIsMuon_column] == self.MuonPIDIsMuon_cut) &
                     (X[self.IsSignalDaughter_column] == self.IsSignalDaughter_cut) &
                     (X[self.TRCHI2DOF_column] < self.TRCHI2DOF_cut) &
                     (X[self.TRGHP_column] < self.TRGHP_cut) &
                     (X[self.PROBNNmu_column] > self.PROBNNmu_cut) &
                     (X[self.PROBNNpi_column] < self.PROBNNpi_cut) &
                     (X[self.PROBNNe_column] < self.PROBNNe_cut) &
                     (X[self.PROBNNk_column] < self.PROBNNk_cut) &
                     (X[self.PROBNNp_column] < self.PROBNNp_cut) &
                     (X[self.IPPUs_column] > self.IPPUs_cut) &
                     (X[self.RecVertexIPs_column] > self.RecVertexIPs_cut))
        X = X[selection]
        if y is not None:
            y = y[selection]

        if self.only_max_pt:
            X.reset_index(drop=True, inplace=True)
            max_pt_indices = (X
                              .groupby(self.event_identifier_column)[self.PT_column]
                              .idxmax())
            X = X.iloc[max_pt_indices]
            if y is not None:
                y.reset_index(drop=True, inplace=True)
                y = y.iloc[max_pt_indices]

        len_after = get_event_number(X)
        self.efficiency_ = len_after / len_before

        if self.mvaFeatures:
            X = X[self.mvaFeatures]

        if y is not None:
            return X, y
        else:
            return X

    def get_params(self, deep=False):
        params = super(CutBasedXGBClassifier, self).get_params(deep=deep)
        for cp in self.cut_parameters:
            params['{}_cut'.format(cp)] = getattr(self,
                                                  '{}_cut'.format(cp))
            params['{}_column'.format(cp)] = getattr(self,
                                                     '{}_column'.format(cp))
        params['mvaFeatures'] = self.mvaFeatures
        params['only_max_pt'] = self.only_max_pt
        params['event_identifier_column'] = self.event_identifier_column
        return params

    def set_params(self, **kwargs):
        for cp in self.cut_parameters:
            cutname = '{}_cut'.format(cp)
            if cutname in kwargs and kwargs[cutname] is not None:
                setattr(self, cutname, kwargs.pop(cutname))
        for other_param in ['mvaFeatures',
                            'only_max_pt',
                            'event_identifier_column']:
            if other_param in kwargs and kwargs[other_param] is not None:
                setattr(self, other_param, kwargs.pop(other_param))
        super(CutBasedXGBClassifier, self).set_params(**kwargs)
        return self

    def fit(self, X, y, X_calib, y_calib, eval_set=None, **kwargs):
        if eval_set is not None:
            raise "Passing an eval_set to xgb ist not yet implemented"
        X, y = self.select(X, y)
        if len(X) < 2:
            return self
        if X_calib is None or y_calib is None:
            X, X_calib, y, y_calib = train_test_split(X, y, test_size=0.5)
        else:
            X_calib, y_calib = self.select(X_calib, y_calib)
        super(CutBasedXGBClassifier, self).fit(X, y, **kwargs)
        probas = super(CutBasedXGBClassifier, self).predict_proba(X_calib)[:, 1]
        self.calibrator.fit(probas.reshape(-1, 1), y_calib)
        self.fit_successful = True
        return self

    def predict_proba(self, data, **kwargs):
        data = self.select(data)
        try:
            uncalibrated = (super(CutBasedXGBClassifier, self)
                            .predict_proba(data, **kwargs))
            return self.calibrator.predict_proba(uncalibrated[:, 1])
        except:
            return np.array([[0.5, 0.5]])

    def score(self, X, y, sample_weight=None):
        probas = self.predict_proba(X)[:, 1]
        sc = tagging_power_score(probas, efficiency=self.efficiency_,
                                 sample_weight=sample_weight)
        return sc

def setup_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('-s', '--slices', type=int, default=None)
    parser.add_argument('-n', '--n-folds', type=int, default=2)
    parser.add_argument('-x', '--xgboost-jobs', type=int, default=1)
    parser.add_argument('-c', '--cv-jobs', type=int, default=1)
    parser.add_argument('-i', '--iter', type=int, default=100)
    parser.add_argument('-y', '--year', type=int, default=2011)
    return parser.parse_args()

def main():
    args = setup_args()

    # style setup
    plt.style.use('ggplot')
    plt.rcParams['figure.figsize'] = (10, 10)
    plt.rcParams['font.size'] = 14

    # just define some keyword arguments for read_root in a separate dict
    chunksize = 5000
    data_kwargs = dict(
        key='DecayTree',  # the tree name
        columns=['B_OS_Muon*',  # all branches that should be read
                 'B_ID',
                 'B_PT',
                 'SigYield_sw',
                 'runNumber',
                 'eventNumber',
                 ],
        chunksize=chunksize,  # this will create a generator, yielding subsets
                              # with 'chunksize' of the data
        where='(B_LOKI_MASS_JpsiConstr_NoPVConstr>0)',  # a ROOT where
                                                        # selection, does not
                                                        # work with
                                                        # array-variables
        flatten=True  # will flatten the data in the dimension of the first
                      # given column
    )

    data_dir = '/home/kheinicke/tank/flavourtagging/'
    year = args.year
    filenames = [
        data_dir + 'Bu2JpsiK_mu-k-e-TrainingTuple_' + str(year) + '_MD_sweighted_kheinick.root',
        data_dir + 'Bu2JpsiK_mu-k-e-TrainingTuple_' + str(year) + '_MU_sweighted_kheinick.root',
        data_dir + 'Bu2JpsiK_mu-k-e-TrainingTuple_' + str(year + 1) + '_MD_sweighted_kheinick.root',
        data_dir + 'Bu2JpsiK_mu-k-e-TrainingTuple_' + str(year + 1) + '_MU_sweighted_kheinick.root',
    ]
    chunksize = 5000

    # still have to use plain ROOT to get the number of entries...
    n_entries = 0
    for fn in filenames:
        f = ROOT.TFile(fn)
        t = f.Get('DecayTree')
        n_entries += t.GetEntries()

    # This will read chunks of the data inside a list comprehension and then
    # concat those to a big dataframe note that tqdm is just some boilerplate
    # to generate a progressbar
    if args.slices is None:
        total = n_entries / chunksize
    else:
        total = args.slices
    print('Reading {} chunks of {}-data'.format(total, year))
    df = pd.concat([df for df in tqdm(
        islice(read_root(filenames, **data_kwargs), args.slices),
        total=total)
    ])

    # prepare dataframe
    df['target'] = np.sign(df.B_ID) == np.sign(df.B_OS_Muon_ID)
    df.rename(columns=dict(zip(df.columns, [c.replace('B_OS_Muon', 'tp')
                                            for c in df.columns])),
              inplace=True)
    df['tp_ABS_RecVertexIP'] = np.abs(df.tp_RecVertexIP)
    df['event_id'] = df.runNumber.apply(str) + '_' + df.eventNumber.apply(str)

    # this is the list of BDT variables formerly used
    classic_MVA_features = ['tp_' + c for c in [
        'partP',
        'partPt',
        'IPPU',
        'ghostProb',
        'PIDNNm',
        'ABS_RecVertexIP',
        'mult',
        'ptB',
        'IPs',
    ]]

    # instead of using the sklearn randomized gridsearch, implement one that
    # handles 3-fold validation
    parameter_grid = OrderedDict([
            ('P_cut', np.linspace(1, 5, 21)),
            ('PT_cut', np.linspace(0, 3, 16)),
            ('phiDistance_cut', np.linspace(0, 0.5, 6)),
            ('TRGHP_cut', np.linspace(0, 0.6, 6)),
            ('IPPUs_cut', np.linspace(1, 4, 16)),
            ('TRCHI2DOF_cut', np.linspace(2, 5, 6)),
            ('PROBNNmu_cut', np.linspace(0.0, 0.6, 6)),
        ])

    # produce all possible grid points
    grid = []
    for cut_values in product(*parameter_grid.values()):
        grid.append({p: v for p, v in zip(parameter_grid.keys(), cut_values)})
    np.random.shuffle(grid)

    # iterate the grid points
    scores = []
    print('Starting grid search')
    for classifier_args in tqdm(grid[:args.iter]):
        # yield 3-fold split for this grid point
        df_sets = [df.iloc[indices] for indices in NSplit(df)]

        cv_scores = []
        for i in range(3):
            df1, df2, df3 = df_sets[i % 3], df_sets[(i + 1) % 3], df_sets[(i + 2) % 3]
            model = CutBasedXGBClassifier(nthread=args.xgboost_jobs,
                                          mvaFeatures=classic_MVA_features,
                                          n_estimators=300,
                                          **classifier_args)
            model.fit(df1, df1.target,
                      df2, df2.target)
            if model.fit_successful:
                cv_scores.append(model.score(df3, df3.target))
            else:
                cv_scores.append(0)

        scores.append((np.mean(cv_scores), cv_scores, classifier_args))

    scores = sorted(scores, key=lambda s: s[0], reverse=True)
    print('tagging power {}% with params\n{}'
          .format(100 * ufloat(scores[0][0], np.std(scores[0][1])),
                  scores[0][2]))
    if not path.isdir('build/'):
        mkdir('build/')
        print('build/ directory created')
    output_filename = ('build/search-{:%Y%m%d-%H%M%S}.json'
                       .format(datetime.datetime.now()))
    with open(output_filename, 'w') as f:
        json.dump({'settings': vars(args),
                   'scores': scores,
                   'parameter-grid': {k: list(v) for k, v in parameter_grid.items()},
                   }, f, indent=4)
        print('grid search results have been written to {}'
              .format(output_filename))

if __name__ == '__main__':
    main()


import pkuseg
import torch
from string import punctuation
from format_utils import format_text, get_train_on_gpu
import sys

# method to predict the sentiment of chinese text

def predict(text, no_stopwords):
    print(f'Text: {text}')

    feature_tensor = format_text(text, no_stopwords)

    batch_size = feature_tensor.size(0)

    # initialize hidden state
    h = net.init_hidden(batch_size)

    # move to gpu if on gpu
    if(get_train_on_gpu()):
        feature_tensor = feature_tensor.cuda()

    # get output from model
    output, h = net(feature_tensor, h)

    # normalize output probs to 0 or 1
    pred = torch.round(output.squeeze())

    # printing output value, before rounding
    print('Prediction value, pre-rounding: {:.6f}'.format(output.item()))

    if pred == 0:
        print('negative')
        return 'Text is negative'
    elif pred == 1:
        print('positive')
        return 'Text is positive'

###########################################

batch_size = 50

net = torch.load('models/torch_rnn_trained.model',
                 map_location=torch.device('cpu'))

predict(sys.argv[1], sys.argv[2])


"""Main CLI method."""
import sys

import click

from eddington import EddingtonException, __version__

class CatchEddingtonExceptions(click.Group):
    """Implementation for a commands group that catches Eddington exceptions."""

    def __call__(self, *args, **kwargs):
        """
        Override of the call method that catches Eddington exceptions.

        :param args: Arguments for the subcommand
        :type args: list
        :param kwargs: Keyword argument for the subcommand.
        :type kwargs: dict
        :return: Returns the output of the subcommand
        """
        try:
            return self.main(*args, **kwargs)
        except EddingtonException as exc:
            click.echo(
                f'{click.style(exc.__class__.__name__, fg="yellow")}: '
                f'{click.style(exc, fg="red")}'
            )
            sys.exit(1)

@click.group("eddington", cls=CatchEddingtonExceptions)
@click.version_option(version=__version__)
def eddington_cli():
    """Command line for Eddington."""