import requests
import argparse
import sys
import socket
from bs4 import BeautifulSoup

KEYWORDS = ("Nazi", "Hitler")

parser = argparse.ArgumentParser(description="Search wikipedia for godwin point")

parser.add_argument("start_page", help="Wikipedia start page")
parser.add_argument("-d", "--depth", help="Depth of the research", action="store", default=5)
args = parser.parse_args()

depth_max = int(args.depth)
soup = {}
links = {}

def print_ident(string, ident):
    print(ident * "    " + string)

def check_page(page, current_depth=0):
    print_ident(f'Looking for Godwin point at "{page}" (depth={current_depth})', current_depth)
    try:
        r = requests.get("https://fr.wikipedia.org" + page)
        soup[page] = BeautifulSoup(r.text, "html.parser")
        text = soup[page].get_text()

    except socket.gaierror:
        print("No internet connection")
        sys.exit(0)

    if any(s in text for s in KEYWORDS):
        print_ident("Godwin point FOUND", current_depth)
        return True
    else:
        if current_depth == depth_max:
            return False
        else:
            print_ident(f"Not found in depth {current_depth}. Looking deeper..", current_depth)
            current_depth += 1
            links[page] = []
            for link in soup[page].find(id="content").find_all("a"):
                if (
                    str(link.get("href")).startswith("/wiki/")
                    and not str(link.get("href")).startswith("/wiki/Fichier:")
                    and not str(link.get("href")).startswith("/wiki/Portail:")
                    and not str(link.get("href")).startswith("/wiki/Cat%C3%A9gorie:")
                    and not str(link.get("href")).startswith("/wiki/Aide:")
                ):
                    links[page].append(link.get("href"))
                    # print(link.get('href'))

            print_ident(f'{len(links[page])} links found in page "{page}"', current_depth)

            for link in links[page]:
                check_page(link, current_depth)

if __name__ == "__main__":
    try:
        check_page("/wiki/" + args.start_page)
    except KeyboardInterrupt:
        print("Bye bye")
        sys.exit(0)


import numpy as np
from scipy import ndimage
def sobelImageSet(images):
    # Apply Sobel Filter in x- and y-direction.
    # Return combined sobelFilter
    sobX = np.zeros(images.shape)
    sobY = np.zeros(images.shape)
    OUT = np.zeros(images.shape)
    for img in range(images.shape[0]):
        sobX[img,:,:] = ndimage.sobel(images[img,:,:],axis=0,mode='constant')
        sobY[img,:,:] = ndimage.sobel(images[img,:,:],axis=1,mode='constant')
        OUT[img,:,:] = np.hypot(sobX[img,:,:],sobY[img,:,:])
        # scale to range [0,1]
        OUT[img,:,:] = (OUT[img,:,:]-np.min(OUT[img,:,:])) / (np.max(OUT[img,:,:])-np.min(OUT[img,:,:]))
    return OUT


import json
import os
import os.path as op
import time
import warnings

import numpy as np

from ._utils import get_args

from ._fetching import fetch_raw_files
from ._scoring import default_score
from ._sss_legacy import push_raw_files, fetch_sss_files
from ._sss import run_sss
from ._fix import fix_eeg_files
from ._ssp import do_preprocessing_combined, apply_preprocessing_combined
from ._epoching import save_epochs
from ._cov import gen_covariances
from ._forward import gen_forwards
from ._inverse import gen_inverses
from ._status import print_proc_status
from ._paths import _get_config_file
from ._utils import timestring

# Class adapted from:
# http://stackoverflow.com/questions/3603502/

class Frozen(object):
    __isfrozen = False

    def __setattr__(self, key, value):
        if self.__isfrozen and not hasattr(self, key):
            raise AttributeError('%r is not an attribute of class %s. Call '
                                 '"unfreeze()" to allow addition of new '
                                 'attributes' % (key, self))
        object.__setattr__(self, key, value)

    def freeze(self):
        """Freeze the object so that only existing properties can be set"""
        self.__isfrozen = True

    def unfreeze(self):
        """Unfreeze the object so that additional properties can be added"""
        self.__isfrozen = False

# noinspection PyUnresolvedReferences
class Params(Frozen):
    """Make a parameter structure for use with `do_processing`.

    See the :ref:`overview` for a description of the options.

    Some params can be set on init, but it's better to set them in
    a YAML file.

    See also
    --------
    do_processing
    read_params
    """

    def __init__(self, tmin=None, tmax=None, t_adjust=0, bmin=-0.2, bmax=0.0,
                 n_jobs=6, lp_cut=55, decim=5, proj_sfreq=None, n_jobs_mkl=1,
                 n_jobs_fir='cuda', n_jobs_resample='cuda',
                 filter_length=32768, drop_thresh=1,
                 epochs_type='fif', fwd_mindist=2.0,
                 bem_type='5120-5120-5120', auto_bad=None,
                 ecg_channel=None, eog_channel=None,
                 plot_raw=False, match_fun=None, hp_cut=None,
                 cov_method='empirical', ssp_eog_reject=None,
                 ssp_ecg_reject=None, baseline='individual',
                 reject_tmin=None, reject_tmax=None,
                 lp_trans=0.5, hp_trans=0.5):
        self.reject = dict(eog=np.inf, grad=1500e-13, mag=5000e-15, eeg=150e-6)
        self.flat = dict(eog=0, grad=1e-13, mag=1e-15, eeg=1e-6)
        self.eog_thresh = None
        self.heog_thresh = None
        self.veog_thresh = None
        if ssp_eog_reject is None:
            ssp_eog_reject = dict(grad=2000e-13, mag=3000e-15,
                                  eeg=500e-6, eog=np.inf)
        if ssp_ecg_reject is None:
            ssp_ecg_reject = dict(grad=2000e-13, mag=3000e-15,
                                  eeg=50e-6, eog=250e-6)
        self.ssp_eog_reject = ssp_eog_reject
        self.ssp_ecg_reject = ssp_ecg_reject
        self.tmin = tmin
        self.tmax = tmax
        self.reject_tmin = reject_tmin
        self.reject_tmax = reject_tmax
        self.t_adjust = t_adjust
        self.baseline = baseline
        self.bmin = bmin
        self.bmax = bmax
        self.run_names = []
        self.inv_names = []
        self.inv_runs = []
        self.work_dir = os.getcwd()
        self.n_jobs = n_jobs
        self.n_jobs_mkl = n_jobs_mkl
        self.n_jobs_fir = n_jobs_fir  # Jobs when using method='fir'
        self.n_jobs_resample = n_jobs_resample
        self.filter_length = filter_length
        self.cont_hp = None
        self.cont_hp_trans = 0.5
        self.cont_lp = 5.
        self.cont_lp_trans = 0.5
        self.hp_cut = hp_cut
        self.lp_cut = lp_cut
        self.hp_trans = hp_trans
        self.lp_trans = lp_trans
        self.phase = 'zero-double'
        self.fir_window = 'hann'
        self.fir_design = 'firwin2'
        self.disp_files = True
        self.plot_drop_logs = False  # plot drop logs after do_preprocessing_
        self.proj_sfreq = proj_sfreq
        self.decim = decim
        self.drop_thresh = drop_thresh
        self.bem_type = bem_type
        self.match_fun = match_fun
        if isinstance(epochs_type, str):
            epochs_type = [epochs_type]
        if not all([t in ('mat', 'fif') for t in epochs_type]):
            raise ValueError('All entries in "epochs_type" must be "mat" '
                             'or "fif"')
        self.epochs_type = epochs_type
        self.fwd_mindist = fwd_mindist
        self.mf_autobad = False
        self.mf_autobad_type = 'maxfilter'
        self.mf_badlimit = 7
        self.auto_bad = auto_bad
        self.auto_bad_reject = None
        self.auto_bad_flat = None
        self.auto_bad_meg_thresh = 10
        self.auto_bad_eeg_thresh = 10
        self.ecg_channel = ecg_channel
        self.eog_channel = eog_channel
        self.heog_channel = 'EOG061'
        self.veog_channel = 'EOG062'
        self.plot_raw = plot_raw

        # add standard file tags

        self.epochs_dir = 'epochs'
        self.cov_dir = 'covariance'
        self.inverse_dir = 'inverse'
        self.forward_dir = 'forward'
        self.list_dir = 'lists'
        self.trans_dir = 'trans'
        self.bad_dir = 'bads'
        self.raw_dir = 'raw_fif'
        self.sss_dir = 'sss_fif'
        self.pca_dir = 'sss_pca_fif'

        self.epochs_tag = '-epo'
        self.inv_tag = '-sss'
        self.inv_fixed_tag = '-fixed'
        self.inv_loose_tag = ''
        self.inv_free_tag = '-free'
        self.inv_erm_tag = '-erm'
        self.eq_tag = 'eq'
        self.sss_fif_tag = '_raw_sss.fif'
        self.bad_tag = '_post-sss.txt'
        self.keep_orig = False
        # This is used by fix_eeg_channels to fix original files
        self.raw_fif_tag = '_raw.fif'
        self.cal_file = 'uw'
        self.ct_file = 'uw'
        # SSS denoising params
        self.sss_type = 'maxfilter'
        self.hp_type = 'maxfilter'
        self.mf_args = ''
        self.tsss_dur = 60.
        self.trans_to = 'median'  # where to transform head positions to
        self.sss_format = 'float'  # output type for MaxFilter
        self.movecomp = 'inter'
        self.int_order = 8
        self.ext_order = 3
        self.st_correlation = .98
        self.sss_origin = 'auto'
        self.sss_regularize = 'in'
        self.filter_chpi = True
        # boolean for whether data set(s) have an individual mri
        self.on_process = None
        # Use more than EXTRA points to fit headshape
        self.dig_with_eeg = False
        # Function to pick a subset of events to use to make a covariance
        self.pick_events_cov = None  # or string 'restrict'
        self.cov_method = cov_method
        self.proj_extra = None
        # These should be overridden by the user unless they are only doing
        # a small subset, e.g. epoching
        self.subjects = []
        self.structurals = None
        self.dates = None
        self.score = None  # defaults to passing events through
        self.acq_ssh = self.acq_dir = None
        self.acq_exclude = []
        self.acq_port = 22
        self.sws_ssh = self.sws_dir = None
        self.sws_port = 22
        self.subject_indices = []
        self.get_projs_from = None
        self.runs_empty = []
        self.proj_nums = [[0] * 3] * 3
        self.in_names = []
        self.in_numbers = []
        self.analyses = []
        self.out_names = []
        self.out_numbers = []
        self.must_match = []
        self.on_missing = 'raise'  # for epochs
        self.subject_run_indices = None
        self.autoreject_thresholds = False
        self.autoreject_types = ['mag', 'grad', 'eeg']
        self.pick_events_autoreject = None  # similar to .pick_events_cov
        self.subjects_dir = None
        self.src_pos = 7.
        self.report_params = dict(
            chpi_snr=True,
            good_hpi_count=True,
            head_movement=True,
            raw_segments=True,
            psd=True,
            ssp_topomaps=True,
            drop_log=None,
            covariance=None,
            snr=None,
            whitening=False,
            sensor=None,
            source_alignment=True,
            bem=True,
            source=None,
            pre_fun=None,
            post_fun=None,
            preload=False,
        )
        self.rotation_limit = np.inf
        self.translation_limit = np.inf
        self.coil_bad_count_duration_limit = np.inf  # for annotations
        self.coil_dist_limit = 0.005
        self.coil_gof_limit = 0.98
        self.coil_t_window = 0.2  # default is same as MF
        self.filter_chpi_t_window = None
        self.coil_t_step_min = 0.01
        self.proj_ave = False
        self.compute_rank = False
        self.cov_rank = 'full'
        self.force_erm_cov_rank_full = True  # force empty-room inv rank
        self.cov_rank_tol = 1e-6
        self.eog_t_lims = (-0.25, 0.25)
        self.heog_t_lims = (-0.25, 0.25)
        self.veog_t_lims = (-0.25, 0.25)
        self.ecg_t_lims = (-0.08, 0.08)
        self.eog_f_lims = (0, 2)
        self.heog_f_lims = (0, 2)
        self.veog_f_lims = (0, 2)
        self.ecg_f_lims = (5, 35)
        self.proj_meg = 'separate'
        self.src = 'oct6'
        self.epochs_prefix = 'All'
        self.reject_epochs_by_annot = True
        self.mf_prebad = dict()
        self.every_other = False
        self.cov_rank_method = 'estimate_rank'
        self.epochs_proj = True
        self.cont_as_esss = False
        self.cont_reject = None
        self.allow_resample = False
        self.freeze()
        # Read static-able paraws from config file
        _set_static(self)

    @property
    def report(self):  # wrapper
        return self.report_params

    @report.setter
    def report(self, report_params):
        self.report_params = report_params

    @property
    def pca_extra(self):
        return '_allclean_fil%d' % self.lp_cut

    @property
    def pca_fif_tag(self):
        return self.pca_extra + self.sss_fif_tag

    def convert_subjects(self, subj_template, struc_template=None):
        """Convert subject names.

        Parameters
        ----------
        subj_template : str
            Subject template to use.
        struc_template : str
            Structural template to use.
        """
        if struc_template is not None:
            if isinstance(struc_template, str):
                def fun(x):
                    return struc_template % x
            else:
                fun = struc_template
            new = [fun(subj) for subj in self.subjects]
            assert all(isinstance(subj, str) for subj in new)
            self.structurals = new
        if isinstance(subj_template, str):
            def fun(x):
                return subj_template % x
        else:
            fun = subj_template
        new = [fun(subj) for subj in self.subjects]
        assert all(isinstance(subj, str) for subj in new)
        self.subjects = new

    def save(self, fname):
        """Save to a YAML file.

        Parameters
        ----------
        fname : str
            The filename to use. Should end in '.yml'.
        """
        from ._yaml import _write_params
        if not (isinstance(fname, str) and fname.endswith('.yml')):
            raise ValueError('fname should be a str ending with .yml, got %r'
                             % (fname,))
        _write_params(fname, self)

def _set_static(p):
    config_file = _get_config_file()
    key_cast = dict(
        sws_dir=str,
        sws_ssh=str,
        sws_port=int,
    )
    if op.isfile(config_file):
        try:
            with open(config_file, 'rb') as fid:
                config = json.load(fid)
        except Exception as exp:
            raise RuntimeError('Could not parse mnefun config file %s, got '
                               'likely JSON syntax error:\n%s'
                               % (config_file, exp))
        for key, cast in key_cast.items():
            if key in config:
                setattr(p, key, cast(config[key]))
    else:
        warnings.warn(
            'Machine configuration file %s not found, for best compatibility '
            'you should create this file.' % (config_file,))
    for key in key_cast:
        if getattr(p, key, None) is None:
            warnings.warn('Configuration value p.%s was None, remote MaxFilter'
                          ' processing will fail.' % (key,))

def do_processing(p, fetch_raw=False, do_score=False, push_raw=False,
                  do_sss=False, fetch_sss=False, do_ch_fix=False,
                  gen_ssp=False, apply_ssp=False,
                  write_epochs=False, gen_covs=False, gen_fwd=False,
                  gen_inv=False, gen_report=False, print_status=True):
    """Do M/EEG data processing.

    Parameters
    ----------
    p : instance of Params
        The parameter structure.
    fetch_raw : bool
        Fetch raw recording files from acquisition machine.
    do_score : bool
        Do scoring.
    push_raw : bool
        Push raw recording files to SSS workstation.
    do_sss : bool
        Run SSS remotely on SSS workstation.
    fetch_sss : bool
        Fetch SSS files from SSS workstation.
    do_ch_fix : bool
        Fix channel ordering.
    gen_ssp : bool
        Generate SSP vectors.
    apply_ssp : bool
        Apply SSP vectors and filtering.
    write_epochs : bool
        Write epochs to disk.
    gen_covs : bool
        Generate covariances.
    gen_fwd : bool
        Generate forward solutions.
    get_inv : bool
        Generate inverses.
    gen_report : bool
        Generate HTML reports.
    print_status : bool
        Print status (determined from file structure).
    """
    from ._report import gen_html_report
    # Generate requested things
    if p.sss_type == 'python':
        push_raw = False
        fetch_sss = False
    bools = [fetch_raw,
             do_score,
             push_raw,
             do_sss,
             fetch_sss,
             do_ch_fix,
             gen_ssp,
             apply_ssp,
             write_epochs,
             gen_covs,
             gen_fwd,
             gen_inv,
             gen_report,
             print_status,
             ]
    texts = ['Pulling raw files from acquisition machine',
             'Scoring subjects',
             'Pushing raw files to remote workstation',
             'Running SSS using %s' % p.sss_type,
             'Pulling SSS files from remote workstation',
             'Fixing EEG order',
             'Preprocessing files',
             'Applying preprocessing',
             'Doing epoch EQ/DQ',
             'Generating covariances',
             'Generating forward models',
             'Generating inverse solutions',
             'Generating HTML Reports',
             'Status',
             ]
    score_fun = p.score if p.score is not None else default_score
    if len(get_args(score_fun)) == 2:
        score_fun_two = score_fun

        def score_fun(p, subjects, run_indices):
            return score_fun_two(p, subjects)
    funcs = [fetch_raw_files,
             score_fun,
             push_raw_files,
             run_sss,
             fetch_sss_files,
             fix_eeg_files,
             do_preprocessing_combined,
             apply_preprocessing_combined,
             save_epochs,
             gen_covariances,
             gen_forwards,
             gen_inverses,
             gen_html_report,
             print_proc_status,
             ]
    assert len(bools) == len(texts) == len(funcs)

    # Only run a subset of subjects

    n_subj_orig = len(p.subjects)

    sinds = p.subject_indices
    if sinds is None:
        sinds = np.arange(len(p.subjects))

    subjects = np.array(p.subjects)[sinds].tolist()

    structurals = p.structurals
    if structurals is not None:
        assert len(structurals) == n_subj_orig
        structurals = np.array(structurals)[sinds].tolist()

    dates = p.dates
    if dates is not None:
        assert len(dates) == n_subj_orig
        dates = [tuple([int(dd) for dd in d]) if d is not None else None
                 for d in [p.dates[si] for si in sinds]]

    decim = p.decim
    if not isinstance(decim, (list, tuple)):
        decim = [decim] * len(p.subjects)
    assert len(decim) == n_subj_orig
    decim = np.array(decim)
    assert decim.dtype.char in 'ild', (decim.dtype.char, decim.dtype)
    assert decim.shape == (len(p.subjects),), decim.shape
    decim = decim[sinds]

    run_indices = p.subject_run_indices
    if run_indices is None:
        run_indices = [None] * len(subjects)
    elif isinstance(run_indices, dict):
        run_indices = [run_indices.get(subject, None) for subject in subjects]
    else:
        run_indices = [run_indices[si] for si in sinds]
        assert len(run_indices) == len(subjects)
    run_indices = [np.array(run_idx) if run_idx is not None
                   else np.arange(len(p.run_names)) for run_idx in run_indices]
    assert all(run_idx.ndim == 1 for run_idx in run_indices)
    assert all(np.in1d(r, np.arange(len(p.run_names))).all()
               for r in run_indices)

    # Actually do the work

    outs = [None] * len(bools)
    for ii, (b, text, func) in enumerate(zip(bools, texts, funcs)):
        if b:
            t0 = time.time()
            print(text + '. ')
            if func is None:
                raise ValueError('function is None')
            if func == fix_eeg_files:
                outs[ii] = func(p, subjects, structurals, dates, run_indices)
            elif func in (gen_forwards, gen_html_report):
                outs[ii] = func(p, subjects, structurals, run_indices)
            elif func == save_epochs:
                outs[ii] = func(p, subjects, p.in_names, p.in_numbers,
                                p.analyses, p.out_names, p.out_numbers,
                                p.must_match, decim, run_indices)
            elif func == print_proc_status:
                outs[ii] = func(p, subjects, structurals, p.analyses,
                                run_indices)
            elif func == gen_covariances:
                outs[ii] = func(p, subjects, run_indices, decim)
            else:
                outs[ii] = func(p, subjects, run_indices)
            print('  (' + timestring(time.time() - t0) + ')')
            if p.on_process is not None:
                p.on_process(text, func, outs[ii], p)
    print("Done")


import pandas as pd

from sklearn.ensemble import RandomForestClassifier

from ml_data import mldata_add
from data_dict import ml_data_url, test_ml_data_url

train_feats = ['num_feat1', 'num_feat2', 'num_feat3', 'cat_feat1', 'cat_feat2', 'cat_feat3']
target_feat = ['Result']

def ml_rf_acc(rf=None):
    train_df = pd.read_csv(ml_data_url)
    X_train, y_train = train_df[train_feats], train_df[target_feat].values.flatten()

    test_df = pd.read_csv(test_ml_data_url)
    X_test, y_test = test_df[train_feats], test_df[target_feat].values.flatten()
    if rf is None:
        rf = RandomForestClassifier(n_estimators=100, min_samples_leaf=2, random_state=42)
    rf.fit(X_train, y_train)
    return rf.score(X_test, y_test)

def ml_rf_pred(ctx):
    test_df = mldata_add(pd.DataFrame([ctx]))
    tmp_df = pd.read_csv(ml_data_url)
    train_df, target = tmp_df[train_feats], tmp_df[target_feat].values.flatten()
    rf = RandomForestClassifier(n_estimators=100, min_samples_leaf=2, random_state=42)
    rf.fit(train_df, target)
    prob = rf.predict_proba(test_df).max()
    pred = rf.predict(test_df)[0]
    return prob, pred

def simple_model_tune():
    n_est = [75, 100, 128, 256]
    min_s_l = [1, 2, 0.5]
    max_d = [None, 1]

    for n in n_est:
        for m_l in min_s_l:
            for d in max_d:
                rf = RandomForestClassifier(n_estimators=n, min_samples_leaf=m_l, max_depth=d, random_state=42)
                sc = ml_rf_acc(rf)
                print('Score: {}, n_est: {}, min_sl: {}, max_d: {}'.format(sc, n, m_l, d))

if __name__ == '__main__':
    print('Testing..')
    simple_model_tune()
    # print(ml_rf_acc())


#!/usr/bin/env python
import os.path
import logging
import yaml
import argparse
from setup_logging import setup_logging
from textclassification import TextClassificationServer

if __name__ == "__main__":
    search_directories = [".", "/etc/textclassification"]
    setup_logging()
    logger = logging.getLogger(__name__)
    parser = argparse.ArgumentParser()
    parser.add_argument("-a", "--address", help="define the address for the server")
    parser.add_argument("-C", "--configuration_file", help="set the configuration file")
    parser.add_argument("-p", "--port", help="define the port number which the server uses to listen")
    parser.add_argument("-t", "--timeout", type=float, help="define the port number which the server uses to listen")
    args = parser.parse_args()

    config_found = False
    cfg = None
    config_file_path = None
    if args.configuration_file:
        config_file_path = args.configuration_file
        if os.path.isfile(config_file_path) and os.access(config_file_path, os.R_OK):
            config_found = True
    else:
        for directory in search_directories:
            config_file_path = os.path.join(directory, "textclassification.yml")
            if os.path.isfile(config_file_path) and os.access(config_file_path, os.R_OK):
                config_found = True
                break
    if config_found:
        with open(config_file_path, 'r') as ymlfile:
            cfg = yaml.load(ymlfile)
    else:
        print("The configuration file is not found.")
        exit(1)

    tcs = TextClassificationServer(cfg=cfg["server"])
    try:
        logger.info("Server start")
        tcs.start(address=args.address, port=args.port, timeout=args.timeout)
    except KeyboardInterrupt as err:
        logger.info("Server quit")


# installed imports
from kornia.geometry.transform.imgwarp import get_perspective_transform, warp_perspective, transform_points
from torchvision.utils import save_image
from torchvision.io import read_image
from torchvision import transforms
import torch
from tqdm.auto import tqdm

from pathlib import Path
from glob import glob
import json

# local imports
from dataloader import BoardDetectorDataset
from models import BoardDetector

def warp(img : torch.Tensor, coords, rotate=0, device='cpu'):
    """
    takes a section of an image (four coordinates) and warps to fill the entire image with it
    Args:
        (torch.Tensor) img: batch of image tensors (B, C, H, W)
        (torch.Tensor) coords: coordinates of the 4 corners torch.Size([4, 2])

    Returns:
        (torch.Tensor) warped batch of image tensors (B, C, H, W)
    """
    _, _, h, w = img.shape

    # new corner coordinates will stretch to fit image window
    new_coords = torch.Tensor([[[0,0], [w, 0], [w, h], [0, h]]]).to(device) # rotate = 0
    if rotate == 90:
        new_coords = torch.Tensor([[[0, h], [0, 0], [w, 0], [w, h]]]).to(device) # rotate = 90
    if rotate == -90:
        new_coords = torch.Tensor([[[w, 0], [w, h], [0, h], [0,0]]]).to(device) # rotate = -90
    if rotate == 180:
        new_coords = torch.Tensor([[[0, 0], [w, 0], [w, h], [0, h]]]).to(device) # rotate = 180

    # compute perspective transform
    M = get_perspective_transform(coords, new_coords)
    # use perspective transform matrix to transform resized image
    warped_img = warp_perspective(img, M, dsize=(h, w))

    return warped_img, M

def warp_points(points, coords, img_size=(320, 320), device='cpu'):
    """

    Args:
        points (torch.Tensor): point locations for every piece (batch, n_points, 2)
        coords (torch.Tensor): (1, 8) corner points of the board (x1, y1, x2, y2, ...)
        img_size (tuple): height and width of image
        device: device

    Returns:
        new points that are warped to img_size

    """
    w, h = img_size
    new_coords = torch.Tensor([[[0, 0], [w, 0], [w, h], [0, h]]]).to(device)
    M = get_perspective_transform(reshape_coords(coords), new_coords)

    return transform_points(M, points)

def generate_warped_board_images(load_folder: str, save_folder: str, size: tuple,
                                 board_detector: BoardDetector = None,
                                 board_data: BoardDetectorDataset = None,
                                 device='cpu'):
    """
    takes all normal board images in a folder and warps them using the warp function (see warp function for more details)

    Args:
        (str) load_folder: folder with normal board images
        (str) save_folder: folder where the warped board images go
        (str) json_file: json file downloaded from labelbox "Chess Board Detection" project (not needed if from_model is True)
        (bool) from_model: if True, prediction of the coordinates of the board image are done by the BoardDetector model
        (str) model_path: path of the BoardDetector model used to detect the coordinates of the board image

    """
    tr = transforms.Resize((320, 320)) # for board detector
    out_tr = transforms.Resize(size)
    raw_image_paths = glob(f'{load_folder}/*jpg')
    for i in tqdm(range(len(raw_image_paths))):
        img_path = raw_image_paths[i]
        filename = Path(img_path).stem
        if board_detector is None:
            assert(board_data != None), "no board_data"
            img, coords = board_data[i]
            raw_inp = (read_image(img_path) / 255.0).unsqueeze(0).to(device)
            coords = reshape_coords(coords * size[0]).to(device)
            out = warp(img=out_tr(raw_inp), coords=coords, device=device)
            save_image(out[0], save_folder + f"/{filename}_warped.jpg")
        else:
            assert(board_detector != None), "no board_detector"
            raw_inp = (read_image(img_path) / 255.0).unsqueeze(0).to(device)
            inp = tr(raw_inp)
            coords = reshape_coords(board_detector(inp)[0] * size[0])
            out = warp(img=out_tr(raw_inp), coords=coords, device=device)
            save_image(out[0], save_folder + f"/{filename}_warped_pred.jpg")

def reshape_coords(out: torch.Tensor) -> torch.Tensor:
    """
    reshapes output of (8,) from BoardDetector to coords of torch.Size([4, 2]), for the warp function
    """
    assert(out.shape == torch.Size([1, 8]) or out.shape == torch.Size([8,])), f"incorrect input size expected ({torch.Size[1, 8]}, got {out.shape}"
    return out.reshape((1, 4, 2))


import time
import asyncio
import logging

from .state import StateController
from .ultrasonic import Ultrasonic

logger = logging.getLogger('executor')

class Executor:
    def __init__(self, config):
        self._control_interval = config['control_interval']
        self.controller = StateController(config)
        self.ultrasonic = Ultrasonic(config['pins']['ultrasonic'])

    async def run(self):
        self.controller.start()
        start_time = time.time()
        logger.debug('control loop started')
        try:
            while True:
                distance = self.ultrasonic.measure()
                self.controller.ultrasonic(distance)

                now = time.time()
                time_remains = start_time + self._control_interval - now
                if time_remains > 0:
                    await asyncio.sleep(time_remains)
                    start_time += self._control_interval
                else:
                    await asyncio.sleep(0) # a chance to cancel
                    start_time = now
                    logger.warning('Overrun control interval by %f seconds', -time_remains)
        finally:
            self.controller.stop()
            logger.debug('control loop exited')


import cv2
import numpy    as np
import matplotlib.pyplot as plt
import pickle
import pandas as pd
import glob
from tqdm import tqdm

def plotFitedEllipse(ccmap,detected_ellipses):
    """
    相関曲面の上位1、5、10パーセントの値の等高線にフィッテイングした楕円と最大値を描画する

    関数中の1.637という数字は1m/sで動く構造が12時間隔てたLIR画像間で移動するおおよその距離をピクセル単位で表したもの。
    以下、ピクセル移動速度と実際の移動速度の換算

    - 金星半径6051kmとして赤道では1周2pi6051km を1440ピクセルで表現している
    - 1 pixelあたり26.389km
    - 1 pixel/12h = 26.389km/12h = 0.61 m/s
    - 1 m/s = 1.637 pixel/12h
    """

    #変数名シノニムを作るのではなく新たにメモリを確保
    ccmap_add_fitted_ellipse =  np.copy(ccmap)

    ellipseText = ' '
    for ellipse in detected_ellipses:
        cv2.ellipse(ccmap_add_fitted_ellipse, (ellipse['center'], ellipse['axes'], ellipse['angle']),-1)
        ellipseText += str(ellipse['threshold']) + 'long axis {:.3g} '.format(ellipse['axes'][1]/1.637)

    fs =22
    fs2=24

    fig, ax = plt.subplots(1, 1, figsize=(10,8))
    mappable = ax.pcolormesh(ccmap_add_fitted_ellipse,cmap='jet',vmin=np.min(ccmap),vmax=np.max(ccmap))

    upper = lambda parcent : np.sort(ccmap.reshape((1,np.array(ccmap).size)))[0][int(parcent/100.*np.array(ccmap).size)]
    ax.contour(ccmap, levels = [upper(90),upper(95),upper(99)],colors=['black','green','blue'])

    tick = [-0.2,0.0,0.2,0.4,0.6]
    # tick = [-0.1,0.0,0.1,0.2,0.3]

    cbar = fig.colorbar(mappable,ticks=tick)
    cbar.set_label('correlation coefficient',fontsize=fs2)
    cbar.ax.set_yticklabels(tick ,fontsize=fs)

    # ax.set_xticks(np.arange(72+1.637*40,72-1.637*40,-1.637*10))
    # ax.set_xticklabels(np.arange(40,-41,-10),fontsize=fs)
    # ax.set_xlabel(r'Eastward velocity / $\cos(\theta)$  (m/s)',fontsize=fs2)
    #
    # ax.set_yticks(np.arange(72+1.637*40,72-1.637*40,-1.637*10))
    # ax.set_yticklabels(np.arange(40,-41,-10),fontsize=fs)
    # ax.set_ylabel('Northward velocity (m/s)',fontsize=fs2)

    #最大値をマーキング
    maxPiY,maxPiX = np.argmax(ccmap)//ccmap.shape[1],np.argmax(ccmap)%ccmap.shape[1]
    ax.scatter(maxPiX,maxPiY,marker='x',c='black',s=300)

    U,V = (maxPiX-72)/1.637,(maxPiY-72)/1.637

    info_text = 'u {:.3g} '.format( U ) + 'v {:.3g} '.format( V ) + ellipseText

    plt.gcf().text(0,0,info_text)

    plt.tight_layout()
    plt.show()
    return info_text

def recoedFitedEllipse(ccmap,detected_ellipses=[],threshold = 99):
    """
    相関曲面の上位1、5、10パーセントの値の等高線に対して行なった楕円フィッテイングの結果（座標、長軸の大きさ、傾き、等高線の値）をまとめる
    """

    upper_ = lambda ccmap,parcent : np.sort(ccmap.reshape((1,np.array(ccmap).size)))[0][int(parcent/100.*np.array(ccmap).size)]
     #輪郭検出のための前処理：2値化
    _, binaryMap = cv2.threshold(ccmap,upper_(ccmap,threshold),1,cv2.THRESH_BINARY)
    contours, _ = cv2.findContours(binaryMap.astype('u1'),cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)

    #楕円近似
    for conter in contours:
        #There should be at least 5 points to fit the ellipse in function 'fitEllipse'
        if conter.shape[0]>5:
            center,axes,angle = cv2.fitEllipse(conter)

        #     中心がマップ内にあるもののみを取り出す
            # if min(center[0],center[1])>0 and center[0]<ccmap.shape[0] and center[1]<ccmap.shape[1]:

            ellipseInfo = {'center':center,
                                'axes':axes,
                                'angle':angle,
                                'threshold':threshold}
            detected_ellipses += [ellipseInfo]

    return detected_ellipses

def maxUV_SR(ccmap):
    """
    相関曲面で最大となる風速を計算する。
    内部パラメータ85は、相関曲面上の座標と実際の風速を換算する値であり、テンプレートエリアとサーチエリアと比較した画像の時間間隔に依存した値（要改善）
    """

    maxPiY,maxPiX = np.argmax(ccmap)//ccmap.shape[1],np.argmax(ccmap)%ccmap.shape[1]
    #2*7*12+1 == 84+1+84
    U,V = (maxPiX-85)/1.637,-(maxPiY-85)/1.637
    return U,V

if __name__ == '__main__':

    # orbit番号で指定。グループ番号はfor文で回す
    orbits = [i for i in range(79,80,1)]
    row = []
    for orbit in orbits:
        target_dir = './output/orbit'+"{0:04d}".format(orbit)+'/'

        for subDir in sorted(glob.glob(target_dir+'/??')):
            if len(glob.glob(subDir+'/refarencesImagesSummary_*.pickle'))>0:
                pathSummary = glob.glob(subDir+'/refarencesImagesSummary_*.pickle')[0]

                with open(pathSummary,'rb') as f:
                    dfSummary = pickle.load(f)

                _, groupNum, time_begin, _, distance_mean, _, _, subsolar_SRlongtitude, _, _, _ = np.array(dfSummary)[0]
                print(groupNum, time_begin, distance_mean, subsolar_SRlongtitude)

                for pathCCmap in sorted(glob.glob(subDir+'/ccmap(*).pickle')):
                    a,b,c = pathCCmap.rfind('('),pathCCmap.rfind(','),pathCCmap.rfind(')')
                    cX,cY = float(pathCCmap[a+1:b]),float(pathCCmap[b+1:c])

                    cLON,cLAT = cX/1440 * 360, (360-cY)/360 * 90
                    cLT = (subsolar_SRlongtitude - cLON + 180)%360 / 15

                    with open(pathCCmap,'rb') as f:
                        ccmap = pickle.load(f)

                    d99 = recoedFitedEllipse(ccmap,detected_ellipses=[],threshold = 99)
                    d95 = recoedFitedEllipse(ccmap,detected_ellipses=[],threshold = 95)
                    d90 = recoedFitedEllipse(ccmap,detected_ellipses=[],threshold = 90)

                    u,v = maxUV_SR(ccmap)

                    if len(d99)>0:
                        longestAxes99 = max(list(map(lambda di :di['axes'][1], d99)))
                    else:
                        longestAxes99 = 0
            #             print(pathCCmap)

                    if len(d95)>0:
                        longestAxes95 = max(list(map(lambda di :di['axes'][1], d95)))
                    else :
                        longestAxes95 = 0

                    if len(d90)>0:
                        longestAxes90 = max(list(map(lambda di :di['axes'][1], d90)))
                    else :
                        longestAxes90 = 0

                    row += [ [orbit ,groupNum, u, v,
                                   cLT,cLAT,cLON,
                                   time_begin, distance_mean,
                                   np.max(ccmap),len(d99),len(d95),len(d90),
                                   longestAxes99,longestAxes95,longestAxes90,
                                   pathCCmap] ]

        #         d99 = recoedFitedEllipse(ccmap[:,:-n],detected_ellipses=[],threshold = 99)
        #         d95 = recoedFitedEllipse(ccmap[:,:-n],detected_ellipses=d99,threshold = 95)
        #         detected_ellipses = recoedFitedEllipse(ccmap[:,:-n],detected_ellipses=d95,threshold = 90)
        #         plotFitedEllipse(ccmap[:,:-n],d99)

    header = ['orbit','groupNum','u','v',
            'lt','lat','lon','time_begin','distance_mean',
            'maxValue','region99','region95','region90',
            'longestAxes99','longestAxes95','longestAxes90',
            'path']

    df = pd.DataFrame(row,columns=header)
    df.to_csv(target_dir+'/../characteristicsCCmap_'+'{0:04d}'.format(orbit)+'.csv')
    with open(target_dir+'/../characteristicsCCmap_'+'{0:04d}'.format(orbit)+'.pickle','wb') as f:
        pickle.dump(df,f,protocol=2)

        #         print(cY,cLAT,cLT,u,v)
        #         print(d99,d95,d90)#を使って特徴量抽出を行う


from dis import disco
from exec.ask_others import ask_other_agent
from trust.artifacts.content_trust.direct_experience import get_combined_direct_experience_for_agent, direct_experience
from datetime import datetime
from loggers.basic_logger import BasicLogger
from models import Scale
import statistics

def recommendation(agent, other_agent, resource_id, scale, logger, discovery, recency_limit):
    """
    Get recommendations on other agent of third agents and average them to one recommendation value.

    :param agent: The agent which calculates the popularity.
    :type agent: str
    :param other_agent: The other agent for which the popularity value is calculated.
    :type other_agent: str
    :param resource_id: The URI of the evaluated resource.
    :type resource_id: str
    :param scale: The Scale object to be used by the agent.
    :type scale: Scale
    :param logger: The logger object to be used by the agent.
    :type logger: BasicLogger
    :param discovery: Addresses of all agents within the scenario.
    :type discovery: dict
    :param recency_limit: A datetime object which is used for "forgetting" old history entries
    :type recency_limit: datetime
    :return: The Recommendation trust value.
    :rtype: float or int
    """

    agents_to_ask = []
    for third_agent in discovery:
        if third_agent != agent and third_agent != other_agent:
            combined = get_combined_direct_experience_for_agent(
                agent, third_agent, logger, recency_limit, scale)
            if combined != None and combined >= scale.minimum_to_trust_others():
                agents_to_ask.append(third_agent)

    recommendations = ask_for_recommendations(
        agent, resource_id, agents_to_ask, scale, logger, discovery, recency_limit)
    return statistics.median(recommendations) if len(recommendations) > 0 else None

def ask_for_recommendations(agent, resource_id, agents_to_ask, scale, logger, discovery, recency_limit):
    """
    Asking all agents to ask about their recommendation on other agent and listing them.

    :param agent: The agent which calculates the popularity.
    :type agent: str
    :param resource_id: The URI of the evaluated resource.
    :type resource_id: str
    :param agents_to_ask: The third agents to ask for recommendation.
    :type agents_to_ask: list
    :param scale: The Scale object to be used by the agent.
    :type scale: Scale
    :param logger: The logger object to be used by the agent.
    :type logger: BasicLogger
    :param discovery: Addresses of all agents within the scenario.
    :type discovery: dict
    :param recency_limit: A datetime object which is used for "forgetting" old history entries
    :type recency_limit: datetime
    :return: The recommendation trust values of third agents on other_agent.
    :rtype: list
    """
    recommendations = []
    message = f"recommendation_{resource_id}_{datetime.strftime(recency_limit, BasicLogger.get_time_format_string())}"
    for third_agent in agents_to_ask:
        remote_ip, remote_port = discovery[third_agent].split(":")
        response = ask_other_agent(remote_ip, int(remote_port), message)
        if response != 'None':
            received_value = float(response)
            combined = get_combined_direct_experience_for_agent(
                agent, third_agent, logger, recency_limit, scale)
            if combined != None:
                recommendations.append(combined * received_value)

    return recommendations

def recommendation_response(agent, resource_id, recency_limit, scale, logger):
    """
    Giving back the direct XP as the recommendation on other agent in the point of view from agent.

    :param agent: The agent which calculates the popularity.
    :type agent: str
    :param resource_id: The URI of the evaluated resource.
    :type resource_id: str
    :param recency_limit: A datetime object which is used for "forgetting" old history entries
    :type recency_limit: datetime
    :param scale: The Scale object to be used by the agent.
    :type scale: Scale
    :param logger: The logger object to be used by the agent.
    :type logger: BasicLogger
    :return: The recommendation trust values of third agents on other_agent.
    :rtype: list
    """
    return direct_experience(agent, resource_id, recency_limit, scale, logger)


from __future__ import (unicode_literals, division, absolute_import,
                        print_function)
import os

from .constants import PYTHON, MAKEOPTS, build_dir, isosx, PREFIX, iswindows
from .utils import run, replace_in_file

def main(args):
    b = build_dir()
    if isosx:
        b = os.path.join(b, 'python/Python.framework/Versions/2.7')
    elif iswindows:
        b = os.path.join(b, 'private', 'python')
    cmd = [PYTHON, 'configure.py', '--no-pyi', '--bindir=%s/bin' % build_dir()]
    sp = 'Lib' if iswindows else 'lib/python2.7'
    inc = 'include' if iswindows else 'include/python2.7'
    cmd += ['--destdir=%s/%s/site-packages' % (b, sp),
            '--sipdir=%s/share/sip' % b,
            '--incdir=%s/%s' % (b, inc)]
    run(*cmd, library_path=True)
    if iswindows:
        run('nmake'), run('nmake install')
    else:
        run('make ' + MAKEOPTS)
        run('make install')
    q, r = build_dir(), PREFIX
    if iswindows:
        q = q.replace(os.sep, os.sep + os.sep)
        r = r.replace(os.sep, os.sep + os.sep)
    p = 'Lib' if iswindows else 'lib/python2.7'
    replace_in_file(os.path.join(b, p, 'site-packages/sipconfig.py'), q, r)


from django.contrib.auth import authenticate, login
from django.contrib.auth.decorators import login_required
from django.shortcuts import render_to_response
from django.shortcuts import render
from django.http import HttpResponseRedirect
from django.template import Context
from django.template import RequestContext
from django.utils.translation import ugettext_lazy as _
from django.contrib.auth.models import User
from django.core.mail import send_mail
from django.conf import settings
from apps.pf.forms import *
from apps.pf.models import *
import random
import string
from datetime import datetime, timedelta
from django.template.loader import get_template
from django.views.decorators.cache import never_cache
from django.forms.models import inlineformset_factory
from django.shortcuts import get_object_or_404
from django.db import transaction

@login_required
@never_cache
def registradocumento(request,documento_id=None):

	if  documento_id == None:
		documento = Documento()
		documento.user = request.user
	else:
		documento = Documento.objects.get(id = documento_id)

	if request.method == 'POST':
		form = DocumentoForm(request.user, request.POST, instance=documento)

		if form.is_valid():
			form.save()

			return HttpResponseRedirect('/pf/lista')
	else:
		form = DocumentoForm(request.user, instance=documento)

	return render(request, 'pf/registradocumento.html', {'form': form, 'id' : documento_id, 'documento': documento})

@login_required
@never_cache
def listadocumenti(request):
	lista = Documento.objects.all()
	return render(request, 'pf/listadocumenti.html',{'lista': lista,})

@login_required
def eliminadocumento(request, id):
	documento = get_object_or_404(Documento, pk=id)
	if documento.user == request.user:
		documento.delete()
		return HttpResponseRedirect('/pf/listadocumenti/')

@login_required
@never_cache
def registracategoria(request,categoria_id=None):

	if  categoria_id == None:
		categoria = Categoria()
		categoria.comune = request.user.profilo.citta
	else:
		categoria = Categoria.objects.get(id = categoria_id)

	if request.method == 'POST':
		form = CategoriaForm(request.user, request.POST, instance=categoria)

		if form.is_valid():
			form.save()

			return HttpResponseRedirect('/pf/listacategorie')
	else:
		form = CategoriaForm(request.user, instance=categoria)

	return render(request, 'pf/registracategoria.html', {'form': form, 'id' : categoria_id, 'categoria': categoria})

@login_required
@never_cache
def listacategorie(request):
	lista = Categoria.objects.all()
	return render(request, 'pf/listacategorie.html',{'lista': lista,})

@login_required
def eliminacategoria(request, id):
	categoria = get_object_or_404(Categoria, pk=id)
	if categoria.comune == request.user.profilo.citta:
		categoria.delete()
		return HttpResponseRedirect('/pf/listacategorie/')


#!/usr/bin/env python3

import pathlib
from os import makedirs
from datetime import datetime
import argparse
import multiprocessing as mp

import tet.constants
from tet.solver_mp import solver_mp
from tet.data_process import createDir

def run():

    parser = argparse.ArgumentParser(description="python3 array_solver.py")
    parser.add_argument('-p', '--data_path', type=pathlib.Path,
                        required=True, help='Path to create data dir.')
    parser.add_argument('-c', '--constants', type=pathlib.Path,
                        required=False, help='Path to constants json file. If not provided then the application will use the default parameters provided by the tet package.')
    parser.add_argument('-n', '--ncpus', default=mp.cpu_count(), type=int, required=False,
                        help='Number of cpus to use in the process pool. Default option is to use all available cpus.')
    parser.add_argument('-m', '--method', default='bins', choices=tet.constants.solver_params['methods'], type=str,
                        required=False, help='Method of optimization to use. Default option is "bins".')

    cmd_args = parser.parse_args()
    data_path = cmd_args.data_path.joinpath(
        f'data_{datetime.now().strftime("%Y_%h_%d_%T")}')

    try:
        if pathlib.Path.exists(cmd_args.data_path):
            createDir(destination=data_path, replace_query=False)
        else:
            print(
                f"Input data path {cmd_args.data_path} does not exist! Creating it."
            )
            raise OSError()
    except OSError:
        makedirs(data_path)

    temporary_tensorflow_params_dict = {
        key: value for key, value in tet.constants.TensorflowParams.items() if key != 'DTYPE'
    }
    temporary_solver_params_dict = {
        key: value for key, value in tet.constants.solver_params.items() if key != 'methods'
    }
    temporary_solver_params_dict['method'] = cmd_args.method

    if cmd_args.constants != None:
        if not pathlib.Path.exists(cmd_args.constants):
            raise OSError('Provided path does not exist.')
        else:
            parameter_dict = tet.constants.loadConstants(cmd_args.constants)

            # check if parameter dicts are correct
            if parameter_dict['constants'].keys() != tet.constants.system_constants.keys() or parameter_dict['tensorflow_params'].keys() != temporary_tensorflow_params_dict.keys():
                raise KeyError(
                    'Parameter json file has altered keys, please run "make" to reconfigure it.'
                )
            else:
                tet.constants.system_constants = {
                    **tet.constants.system_constants, **parameter_dict['constants']
                }
                tet.constants.TensorflowParams = {
                    **tet.constants.TensorflowParams, **parameter_dict['tensorflow_params']
                }
                tet.constants.solver_params = {
                    **tet.constants.system_constants, **parameter_dict['solver_params']
                }
                lims = parameter_dict['limits']
    else:
        print('\n* No json file was provided, using default parameters. *\n')
        lims = [[-10, 10]
                for _ in range(tet.constants.system_constants['sites'])]

    # Set sites to optimize
    keys = [f'x{k}lims' for k in tet.constants.TensorflowParams['train_sites']]
    trainable_vars_lims = dict(zip(keys, lims))

    result = solver_mp(
        const=tet.constants.system_constants,
        trainable_vars_limits=trainable_vars_lims,
        lr=tet.constants.TensorflowParams['lr'],
        beta_1=tet.constants.TensorflowParams['beta_1'],
        amsgrad=tet.constants.TensorflowParams['amsgrad'],
        target_site=tet.constants.solver_params['target'],
        data_path=data_path,
        method=cmd_args.method,
        write_data=True,
        cpu_count=cmd_args.ncpus
    )

    final_parameters = {
        'constants': result,
        'tensorflow_params': temporary_tensorflow_params_dict,
        'solver_params': temporary_solver_params_dict
    }

    tet.constants.dumpConstants(
        dictionary=final_parameters, path=data_path, name='parameters'
    )

if __name__=="__main__":
    run()


#!/usr/bin/python

import requests
import datetime
import random
import os, sys
import time
import string

def randomString(stringLength):
    letters = string.ascii_letters
    return ''.join(random.choice(letters) for i in range(stringLength))

if len(sys.argv) != 2:
    exit("usage: {} addr".format(sys.argv[0]))

for user_id in range(1):
    payload = {
        "promiseid": randomString(20),
        "description" : "Как меня зовут?",
        "userid": 346,
        'metrics': "",
        'category': 0,
        'wall_pub': True,
        'story_pub': False,
        'exp_date': 0,
        'pub_date': 0,
        'transactions': []
    }
    print(payload)
    r = requests.post("https://" + sys.argv[1] + "/submitpromise", cert='stunnel.pem', json=payload, verify=False)
    print(r)


#!-*- coding:utf8 -*-
# 作者 yanchunhuo
# 创建时间 2018/01/19 22:36
# github https://github.com/yanchunhuo
import ujson
import re
import random
import string
import uuid

class StrTool:
    letters = list(string.ascii_letters)
    whitespace = list(string.whitespace)
    punctuation = list(string.punctuation)
    digits = list(string.digits)
    # 汉字编码的范围
    ch_start = 0x4E00
    ch_end = 0x9FA5

    @classmethod
    def getStringWithLBRB(cls, sourceStr, lbStr, rbStr, offset=0):
        """
        根据字符串左右边界获取内容
        offset:要获得匹配的第几个数据,默认第一个
        :param sourceStr:
        :param lbStr:
        :param rbStr:
        :param offset:
        :return:
        """
        regex = '([\\s\\S]*?)'
        r = re.compile(lbStr + regex + rbStr)
        result = r.findall(sourceStr)
        if str(offset) == 'all':
            return result
        else:
            if len(result) >= offset and len(result) != 0:
                return result[offset]
            else:
                return None

    @classmethod
    def addUUID(cls,prefix:str=None,suffix:str=None,separator:str='_'):
        """_summary_

        Args:
            prefix (_type_, optional): _description_. Defaults to None.
            suffix (_type_, optional): _description_. Defaults to None.
            separator (str, optional): _description_. Defaults to None.

        Returns:
            _type_: _description_
        """
        result=str(uuid.uuid4())
        if prefix:
            result=prefix+separator+result
        if suffix:
            result=result+separator+suffix
        return result

    @classmethod
    def objectToJsonStr(cls, object):
        """
        将类对象转为json字符串
        :param object:
        :return:
        """
        import json
        return json.dumps(object,default=lambda o: o.__dict__)

    @classmethod
    def objectToJson(cls, object):
        """
        将类对象转为json
        :param object:
        :return:
        """
        import json
        return ujson.loads(json.dumps(object,default=lambda o: o.__dict__))

    @classmethod
    def getSpecifiedStr(cls, length, char):
        """
        根据字符获取指定长度的字符串
        :param length:
        :param char:
        :return:
        """
        result = ''
        for i in range(int(length)):
            result = result + str(char)
        return result

    @classmethod
    def addFix(cls, sourceStr, isPre=False, preStr='', isSuffix=False, suffixStr=''):
        """
        字符串加前后缀
        :param sourceStr:
        :param isPre:
        :param preStr:
        :param isSuffix:
        :param suffixStr:
        :return:
        """
        preStr = str(preStr).strip()
        suffixStr = str(suffixStr).strip()
        if isPre and isSuffix:
            return '{}{}{}'.format(preStr, sourceStr, suffixStr)
        elif isSuffix:
            return '{}{}'.format(sourceStr, suffixStr)
        elif isPre:
            return '{}{}'.format(preStr, sourceStr)
        else:
            return sourceStr

    @classmethod
    def getRandomChar(cls):
        """
        随机获取a-zA-Z的单个字符
        :return:
        """
        str = string.ascii_letters
        return random.choice(str)

    @classmethod
    def replaceContentWithLBRB(cls, content, new, lbStr, rbStr, replaceOffset=0):
        """
        根据左右字符串匹配要替换的内容，支持多处匹配只替换一处的功能
        :param content:
        :param new: 要替换的新字符串
        :param lbStr: 要替换内容的左侧字符串
        :param rbStr: 要替换内容的右侧字符串
        :param replaceOffset: 需要将第几个匹配的内容进行替换，下标从0开始，所有都替换使用-1
        :return:
        """
        if lbStr == '' and rbStr == '':
            return
        regex = '([\\s\\S]*?)'
        r = re.compile(lbStr + regex + rbStr)

        match_results = r.findall(content)
        if int(replaceOffset) == -1:
            for result in match_results:
                # 为了防止匹配的内容在其他地方也有被替换掉，故需要将匹配的前后字符串加上
                content = content.replace(lbStr + result + rbStr, lbStr + new + rbStr)
        elif len(match_results) >= replaceOffset and len(match_results) != 0:
            # 用于记录匹配到关键字的位置
            index = None
            for i in range(len(match_results)):
                if i == 0:
                    # 第一次查找匹配所在的位置
                    index = content.find(lbStr + match_results[i] + rbStr)
                else:
                    # 从上一次匹配的位置开始查找下一次匹配的位置
                    index = content.find(lbStr + match_results[i] + rbStr, index + 1)
                if i == int(replaceOffset):
                    preContent = content[:index]
                    centerContent = lbStr + new + rbStr
                    suffContent = content[index + len(lbStr + match_results[i] + rbStr):]
                    content = preContent + centerContent + suffContent
                    break
        return content

    @classmethod
    def random_index(cls, percents):
        """
        随机变量的概率函数,返回概率事件的下标索引
        :return:
        """
        start = 0
        index = 0
        randnum = random.randint(1, sum(percents))

        for index, scope in enumerate(percents):
            start += scope
            if randnum <= start:
                break
        return index

    @classmethod
    def getRandomText(cls, length, ch_percent=90, en_percent=5, digits_percent=3, punctuation_percent=2,
                      whitespace_percent=0):
        """
        获取指定长度文本内容，可设置中文、英文、数字、标点符号、空白字符现的概率
        如果字符串包含中文，返回的内容为Unicode
        :param length: 生成文本的长度
        :param ch_percent: 出现中文字符的概率
        :param en_percent: 出现英文字符的概率
        :param digits_percent: 出现数字字符的概率
        :param punctuation_percent: 出现标点符号的概率
        :param whitespace_percent: 出现空白字符的概率
        :return:
        """
        percents = [ch_percent, en_percent, digits_percent, punctuation_percent, whitespace_percent]
        percents_info = ['ch_percent', 'en_percent', 'digits_percent', 'punctuation_percent', 'whitespace_percent']
        result = ''
        for i in range(length):
            info = percents_info[cls.random_index(percents)]
            if info == 'ch_percent':
                result += chr(random.randint(int(cls.ch_start), int(cls.ch_end)))
            elif info == 'en_percent':
                result += random.choice(cls.letters)
            elif info == 'digits_percent':
                result += random.choice(cls.digits)
            elif info == 'punctuation_percent':
                result += random.choice(cls.punctuation)
            elif info == 'whitespace_percent':
                result += random.choice(cls.whitespace)
        return result

    @classmethod
    def contentToDict(cls, content: str,result_enter_type:str='\r\n'):
        """
        将包含换行符的字符串内容转为字典，目前仅支持格式:key=value
        @param content:
        @param result_enter_type:存储的换行类型，包括\r\n、\n、\r
        @return: {‘key’:{'value':value,'desc':desc}}
        """
        content = content.replace('\r\n', '\n')
        lines = content.split('\n')
        result_dict = {}
        tmp_key_desc=''
        for i,line in enumerate(lines):
            if not line.startswith('#') and not line.startswith('//') and '=' in line:
                tmp_line = line.split('=')
                result_dict.update({tmp_line[0].strip(): {'value':tmp_line[1].strip(),'desc':tmp_key_desc}})
                tmp_key_desc=''
            else:
                tmp_key_desc+=line
                if not i==len(lines)-1:
                    tmp_key_desc+=result_enter_type
        return result_dict

    @classmethod
    def dictToContent(cls, content_dict:dict,result_enter_type:str='\r\n'):
        """
        将def contentToDict(cls, content: str,result_enter_type:str='\r\n')返回的结果拼接为content
        key和value使用=拼接
        @param content_dict:
        @param result_enter_type:存储的换行类型，包括\r\n、\n、\r
        @return:
        """
        result_content = ''
        for key in content_dict.keys():
            result_content += content_dict[key]['desc']
            result_content += key
            result_content += '='
            result_content += content_dict[key]['value']
            result_content += result_enter_type
        return result_content


import tushare as ts
import os
import time

"""
# 设计一个股票、基金池，监控价格
# 从tushare pro获取数据，绘制股票走势图
# 设定一些监控指标进行计算，例如：
    一个月内最低点、三个月内最低点、半年内最低点、一年内最低点
    一个月内最高点、三个月内最高点、半年内最高点、一年内最高点
    一个月内涨跌幅、三个月内涨跌幅、半年内涨跌幅、一年内涨跌幅
    局部最低点、局部最高点
# 成交量的变化
# 不同时间段的趋势类型，例如震荡、上升、下降
"""

# 使用time获取今天的日期
today = time.strftime('%Y%m%d', time.localtime(time.time()))

# df = ts.get_hist_data('600519', start='2020-01-01', end=today)
# print(df)

# 设置tushare pro的token，用于获取数据
ts.set_token(os.getenv("TS_PRO_TOKEN"))

# 初始化pro接口
pro = ts.pro_api()

# 获取贵州茅台过去三年的日线数据
df = pro.daily(ts_code='600519.SH', start_date='20180101', end_date=today)

# 找到最低点
lowest_price = df['close'].min()
lowest_date = df.loc[df['close'].idxmin(), 'trade_date']

# 输出结果
print('最低价:', lowest_price)
print('日期:', lowest_date)


import cv2
import sys
import os
from matplotlib import pyplot as plt

# On effectue les tests sur tous les classifieurs créés (mymaskedfacedetetor 0, 1, 2)
classCascade = cv2.CascadeClassifier('mymaskedfacedetector2.xml')

"""
Premiers tests avec des images venant de différentes sources
Affichage des rectangles pour vérifier la localisation du masque détecté
"""

imagePath = '0-with-mask - Copie.jpg' #modifiable

# Affichage de l'image
image = cv2.imread(imagePath)
plt.imshow(image)
plt.show()

# Detection
faces = classCascade.detectMultiScale(
    image,
    scaleFactor=1.1,
    minNeighbors= 35, #paramètres optimaux donnés ligne 103
    minSize=(30, 30),
    flags = cv2.CASCADE_SCALE_IMAGE
)
print("Il y a {0} visage(s) masqué(s).".format(len(faces)))

# Dessine des rectangles autour des visages masqués trouvés
for (x, y, w, h) in faces:
    cv2.rectangle(image, (x, y), (x+w, y+h), (0, 255, 0), 2)
plt.imshow(image)
plt.show()

"""
Recherche du nombre de minNeighbors optimal.
On cherche nimNeighbors tel que le nombre de visages masqués détecté
soit le plus proche possible du nombre de visage
"""

def Optimal(classifieur_path,data_path):
    classCascade = cv2.CascadeClassifier(classifieur_path)
    files_test = os.listdir(data_path)

    minN_list = [i for i in range(5,100)]
    erreur_list = []

    minErreur = 100
    minNoptimal = 0
    i=0
    for minN in range(5,100):  #à changer selon le classifieur

        erreur = 0

        for name in files_test :
            image = cv2.imread(data_path+'/'+name)
            # Detection
            faces = classCascade.detectMultiScale(
                image,
                scaleFactor=1.1,
                minNeighbors= minN,
                minSize=(30, 30),
                flags = cv2.CASCADE_SCALE_IMAGE
                )

            if len(faces) > 1 :
                erreur = erreur + len(faces) - 1
            elif len(faces) == 0 :
                erreur += 1

        erreur_list.append(erreur)

        if erreur <= minErreur :

            minErreur = erreur
            minNoptimal = minN
    return (minN_list, erreur_list, minErreur, minNoptimal)

# minN_list, erreur_list, minErreur, minNoptimal = Optimal('mymaskedfacedetector2.xml',  'data/positive/positive_test') #paramètres variables

# plt.plot(minN_list, erreur_list)
# plt.show()

"""
Les paramètres optimaux trouvés sont :
mymaskedfacedetector0.xml -> 3
mymaskedfacedetector1.xml -> 7
mymaskedfacedetector2.xml -> 53
"""

"""
(1) Test sur des images où le masque a été ajouté avec photoshop
('data/positive/positive_test')

(2) Test sur des images de visages masqués avec des masques variés
(différents de celui utilisé pour l'entraînement)
 ('data/positive/positive_varied_masks_test')
"""

path_test = 'data/positive/positive_varied_masks_test' #modifiable

files_test = os.listdir(path_test)

i = 0
presence_mask = 0
sum_masks = 0
max_masks = []
images_bug = []
names_images_bug = []
for name in files_test :

    image = cv2.imread(path_test+'/'+name)
    # Detection
    faces = classCascade.detectMultiScale(
        image,
        scaleFactor=1.1,
        minNeighbors=5, #prendre le minNeighbors optimal selon le classifieur
        minSize=(30, 30),
        flags = cv2.CASCADE_SCALE_IMAGE
        )

    # # Affichage des rectangles
    # for (x, y, w, h) in faces:
    #     cv2.rectangle(image, (x, y), (x+w, y+h), (0, 255, 0), 2)
    # plt.imshow(image)
    # plt.show()

    # Analyse de la performance
    i+= 1
    sum_masks = sum_masks + len(faces)
    if len(faces)>= 1 :
        presence_mask += 1

    # On cherche les bugs
    if len(faces)> 1 :
        max_masks.append(len(faces)) #par curiosité on regarde toutes les images 'mal traitrées'
        bug = image
        images_bug.append(bug)
        names_images_bug.append(name)

print("Il y a {0} visage(s) masqué(s) détéctés sur {1} images traitées.".format(sum_masks,i))
print("La présence de masque(s) a été observée sur {0} images parmi les sur {1} images traitées.".format(presence_mask,i))
print("Le score du classifieur est donc de {0}".format(presence_mask/i))
print("Les nombres maximum de visages masqués sur une photo sont de {0}.".format(max_masks))

"""
# Affichage des images bug
for image in images_bug :
    plt.imshow(image)
    plt.show()
"""

"""
Test sur des images négatives (sans visages masqués)
"""

"""
(3) On en profitera pour remarquer la réaction du classifieur pour des images avec masque seul.
On regarde donc, sur un petit jeu de données,
si le classifieur détecte le masque seul.
On remarque que le classifieur réagit bien, il ne considère pas l'objet 'masque' seul.
"""
#path_test = 'data/negative/negative_masques_seuls'

"""
(4) On teste ensuite sur des images variées : visages sans masques / fruits / voitures ...
"""

path_test = 'data/negative/negative_test'

files_test = os.listdir(path_test)

i = 0
presence_mask = 0
sum_masks = 0
max_masks = []
images_bug = []
names_images_bug = []
for name in files_test :

    image = cv2.imread(path_test+'/'+name)
    # Detection
    faces = classCascade.detectMultiScale(
        image,
        scaleFactor=1.1,
        minNeighbors=53,
        minSize=(30, 30),
        flags = cv2.CASCADE_SCALE_IMAGE
        )

    # Analyse de la performance
    i+= 1
    sum_masks = sum_masks + len(faces)
    if len(faces)>= 1 :
        presence_mask += 1

    # On cherche les bugs
    if len(faces)>= 1 :
        max_masks.append(len(faces)) #par curiosité on regarde toutes les images 'mal traitrées'
        bug = image
        images_bug.append(bug)
        names_images_bug.append(name)

print("Il y a {0} visage(s) masqué(s) détéctés sur {1} images traitées.".format(sum_masks,i))
print("La présence de masque(s) a été observée sur {0} images parmi les sur {1} images traitées.".format(presence_mask,i))
print("Le score du classifieur est donc de {0}".format(1 - presence_mask/i))
print("Le nombre maximum de visages masqués sur une photo est de {0}.".format(max_masks))

# #Affichage des images bug
# for name in names_images_bug :
#     image = cv2.imread(path_test+'/'+name)
#     # Detection
#     faces = classCascade.detectMultiScale(
#         image,
#         scaleFactor=1.1,
#         minNeighbors=5,
#         minSize=(30, 30),
#         flags = cv2.CASCADE_SCALE_IMAGE
#         )
#     #Affichage des rectangles
#     for (x, y, w, h) in faces:
#         cv2.rectangle(image, (x, y), (x+w, y+h), (0, 255, 0), 2)
#     plt.imshow(image)
#     plt.show()


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA, NMF
import os

os.chdir(r"D:\1. stark\anaconda_workspace\no.2\머신러닝 알고리즘과 응용\data")

df = pd.read_csv('data_number_nine.csv', header='infer',encoding='latin1')

print(df.shape)

df.head(5)

def ShowMe(X):
    Y= 1.0 - X
    plt.imshow(Y, cmap='gray')
    plt.show()

X = np.array(df)
ShowMe(X)
print(X)

def reducedPCA(X,nPC):
    pca = PCA(n_components = nPC)
    X_pca = pca.fit_transform(X)
    print(X_pca)
    return pca.inverse_transform(X_pca)

for nPC in [23, 10, 5, 3, 1]:
    Z = reducedPCA(X,nPC)
    # print(Z)
    print( "N# of PCs = " + str(nPC))
    ShowMe(Z)
# print(Z)


# -*- coding: utf-8 -*-
"""
Created on Mon Dec 16 22:24:44 2019

@author: z003vrzk
"""

# Python imports
import sys
import os

# Third party imports
import numpy as np
from sklearn.metrics import (auc_roc_score, recall_score,
                             balanced_accuracy_score, precision_score,
                             recall_score, accuracy_score)

# Local imports
if __name__ == '__main__':
    # Remove the drive letter on windows
    _CWD = os.path.splitdrive(os.getcwd())[1]
    _PARTS = _CWD.split(os.sep)
    # Project dir is one level above cwd
    _PROJECT_DIR = os.path.join(os.sep, *_PARTS[:-1])
    if _PROJECT_DIR not in sys.path:
        sys.path.insert(0, _PROJECT_DIR)

#%%

y_true = np.array([0, 1, 0, 0, 1, 0])
y_pred = np.array([0, 1, 0, 0, 0, 1])
weight = np.array([1, 1, 1, 1, 1, 1])

balanced_accuracy_score(y_true, y_pred)

# Balanced accuracy is the mean of sensitivity and specificity
true_positive = (y_pred == 1) & (y_true == 1)
false_negative = (y_pred == 0) & (y_true == 1)
false_positive = (y_pred == 1) & (y_true == 0)
true_negative = (y_pred == 0) & (y_true == 0)
sensitivity = sum(true_positive) / (sum(true_positive) + sum(false_negative))
specificity = sum(true_negative) / (sum(true_negative) + sum(false_positive))

recall0 = recall_score(y_true, y_pred, pos_label=0)
recall1 = recall_score(y_true, y_pred, pos_label=1)

#%% Testing metrics

y_true = np.array(['ahu','alarm','boiler','chiller','chiller',
                   'exhaust_fan','ahu','ahu','ahu'])
y_pred = np.array(['alarm','ahu','boiler','chiller','chiller',
                   'exhaust_fan','ahu','ahu','ahu'])

# Test accuracy, precision, recall
acc = accuracy_score(y_true, y_pred) # 7 / 9 are correct

"""Precision is ability of estimator to give positive labels to instances
not identified as positive
Precision = TP / (TP + FP)
TP = 7
FP = 2"""
prec = precision_score(y_true, y_pred, average='weighted')

"""Recall is ability of estimator to identify all positive classes
Recall = TP / (TP + FN)
TP = 7
FN = 2"""
rec = recall_score(y_true, y_pred, average='weighted')
rec2 = recall_score(y_true, y_pred, average='weighted', labels=['ahu'])

sum(y_true == y_pred) / len(y_true)


import torch
import numpy as np
import hydra
import torch.utils.data as data
import os
from utils.virtualview_loader import VirtualviewScannetDataset
from utils.realview_loader import RealviewScannetDataset, collate_image
from utils.fusion_2d_3d import Fusioner
from utils.miou import miou_2d, miou_3d
from tqdm import tqdm

class Pipeline3D:
    """
    A pipeline supports 3D inference.
    """
    def __init__(self, cfg):
        self.cfg = cfg
        print('Use visdom:', cfg.visdom.use)
        print('Use virtual view:', not cfg.dataset.real_view)

    def inference(self):
        """
        Use 2D segmentation model to do 3D inference, based on Fusion Algorithm.
        """
        device = self.get_device()

        model_path = self.cfg.evaluation.model_path

        model = self.get_model(model_path).to(device)

        scene_dataset = self.get_scene_dataset(
            mode='train',
            use_transform=True)

        # for scene_id in range(len(scene_dataset)):
        for scene_id in range(1):
            single_scene_data = scene_dataset[scene_id]

            image_dataset = single_scene_data['imgset']
            pc_xyz = single_scene_data['point_cloud_xyz'].to(device)
            pc_semantic_label = single_scene_data['semantic_label'].to(device)
            try:
                intrinsic_depth = single_scene_data['intrinsic_depth'].to(device)
            except:
                intrinsic_depth = single_scene_data['intrinsic'].to(device)

            # img loader
            image_dataloader = data.DataLoader(
                dataset=image_dataset,
                batch_size=self.cfg.data_loader.batch_size,
                shuffle=False,
                num_workers=self.cfg.data_loader.num_workers,
                collate_fn=collate_image)

            # a machine able to fuse 2d and 3d
            collection_method = 'knn'
            fusion_machine = Fusioner(point_cloud=pc_xyz,
                                intrinsic_depth=intrinsic_depth,
                                collection_method=collection_method, # average, knn
                                use_gt=False)

            # collect features
            # process_bar = tqdm(total=len(image_dataset)) # show process
            batch_size = self.cfg.data_loader.batch_size

            for idx, batch in enumerate(image_dataloader): # loop over image
                with torch.no_grad():
                    imgs = batch['color_img'].to(device)
                    depth_imgs = batch['depth_img']
                    pose_matrixs = batch['pose_matrix']
                    semantic_labels = batch['semantic_label'].to(device)

                    preds = model(imgs)

                    preds = torch.softmax(preds, dim=1)

                    for i in range(imgs.shape[0]): #single data point
                        depth_img = depth_imgs[i].to(device)
                        img = imgs[i].to(device)
                        pose_matrix = pose_matrixs[i].to(device)
                        pred = preds[i].to(device)

                        fusion_machine.projection(depth_img=depth_img,
                                                pose_matrix=pose_matrix,
                                                feature_img=pred,
                                                threshold=5)

                # process_bar.update(batch_size)
            if collection_method=='average':
                pc_features = fusion_machine.get_features()
                pc_pred_label = torch.max(pc_features, dim=1).indices
            elif collection_method=='knn':
                pc_features = fusion_machine.get_features()
                pc_pred_label = torch.tensor(pc_features)

            # get MIOU
            MIOU = miou_3d(pc_pred_label, pc_semantic_label)

            print("MIOU:", MIOU)

    def get_model(self, model_path=None):
        # load model
        if self.cfg.model.model_name=='deeplabv3+':
            from modeling.deeplab import DeepLab
            backbone = self.cfg.model.backbone
            num_classes = self.cfg.model.num_classes
            output_stride = self.cfg.model.output_stride
            model = DeepLab(backbone=backbone, num_classes=num_classes,
                            output_stride=output_stride)
        elif self.cfg.model.model_name=='unet':
            from unet import UNet
            num_channels = self.cfg.model.num_channels
            num_classes = self.cfg.model.num_classes
            model = UNet(n_channels=num_channels, n_classes=num_classes)

        # load pretrained model
        if model_path is None:
            if self.cfg.model.pretrain:
                pretrain_path = self.cfg.model.pretrained_model_path
                model.load_state_dict(torch.load(pretrain_path))

        # load self path
        if model_path is not None:

            model.load_state_dict(torch.load(model_path))

        print('Model name:', self.cfg.model.model_name)
        return model

    def get_device(self):
        device = None
        if self.cfg.device!='cpu':
            device = torch.device(self.cfg.device)
        else:
            device = torch.device('cpu')
        return device

    def get_scene_dataset(self, mode='train', use_transform=True):
        if self.cfg.dataset.real_view: # use real view
            dataset = RealviewScannetDataset(self.cfg, mode=mode, use_transform=use_transform)
        else: # use virtual view
            dataset = VirtualviewScannetDataset(self.cfg, mode=mode, use_transform=use_transform)

        return dataset

@hydra.main(config_path='config', config_name='config')
def main(cfg):
    ppl = Pipeline3D(cfg)
    ppl.inference()

if __name__=='__main__':
    main()

{'DeepLab': 'modeling.deeplab.DeepLab', 'UNet': 'unet.UNet'}


from gtts import gTTS
import os
from newspaper import Article
from bs4 import BeautifulSoup as bs
import requests
import sqlite3
from sqlite3 import Error

# DB funcs
def create_connection(path):
    """Creates connection to the SQLite database"""
    conn = None
    try:
        conn = sqlite3.connect(path)
        print("Connection to SQLite DB successful")
    except Error as e:
        print(f"The error '{e}' occurred")
    return conn

def execute_query(conn, query, data_tuple=()):
    """Executes SQL query for creating/populating a table"""
    cursor = conn.cursor()
    try:
        cursor.execute(query, data_tuple)
        conn.commit()
        if query == create_books_table:
            print(f"Table was created successfully")
        elif query == insert_into_books_table:
            print(f"Values were inserted successfully")
        else:
            print(f"Query {query} executed successfully")
    except Error as e:
        print(f"The error '{e}' occurred")

create_books_table = """
CREATE TABLE IF NOT EXISTS books (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    book_title TEXT NOT NULL UNIQUE ON CONFLICT FAIL,
    url_address TEXT NOT NULL,
    author_name TEXT NOT NULL);
"""
insert_into_books_table = """
INSERT INTO books
    (book_title, url_address, author_name)
    VALUES (?, ?, ?);
"""

def get_titles():
    """Executes SELECT query to show a book list"""
    with conn:
        cursor.execute("""SELECT id, book_title, author_name FROM books""")
        result = cursor.fetchall()
        for line in result:
            print(f'{line[0]}) {line[1]} (by {line[2]})')

def user_choise():
    """"Choose a book from list"""
    user_input = input('Please choose a book title inputting the number: ')
    try:
        with conn:
            cursor.execute("""SELECT * FROM books WHERE id = ?""", [int(user_input)])
            result = cursor.fetchone()
            user_chosen_book = result[1]
            book_url = result[2]
            book_author = result[3]
            print(f'You have chosen "{user_chosen_book}" written by {book_author}')
            return user_chosen_book, book_url, book_author
    except:
        print("Can't find the book in my database. Try again")
        return user_choise()

# parse funcs
def title_url_author_parser():
    """"Parse book title, url address and author name and insert into SQL Database"""
    for a in soup.findAll('a', href=True, title=True):
        book_title = a['title']
        url = a['href']
        try:
            author_name = bs(requests.get(url).content, "lxml").find('span', class_='entry-tags').text
        except:
            author_name = 'unknown author'
        execute_query(conn, insert_into_books_table, (book_title, url, author_name))
    print('The table is now complete')

def text_parser(book_info):
    """"Parse user chosen book summary"""
    book_summary = Article(book_info[1], language='en')
    book_summary.download()
    book_summary.parse()
    book_text = f'{book_info[0]}\nby {book_info[2]}\n\n{book_summary.text}'
    return book_text

# funcs for making text and audio files
def make_text(title, content):
    """"Make text file"""
    filename = f'Summaries/{title.lower().replace(" ", "_")}.txt'
    os.makedirs(os.path.dirname(filename), exist_ok=True)
    with open(filename, 'w', encoding="utf-8") as text_file:
        text_file.write(content)
    print('The text file was successfully saved ')

def make_audio(title, content):
    """"Make audio file"""
    speech = gTTS(text=content, lang='en')
    filename = f'Audios/{title.lower().replace(" ", "_")}.mp3'
    os.makedirs(os.path.dirname(filename), exist_ok=True)
    print('Saving the audio file. Please wait for a while')
    speech.save(filename)
    print('The audio file was successfully saved')
    # os.system(filename)

# user input funcs
def choose_another_book():
    """"Let user choose another book from a list"""
    while True:
        user_answer = input('Would you like to pick another book? Input yes/no: ')
        if user_answer.lower() in ['нет', 'no']:
            print("Got it. I'm finishing up the program. Good luck!")
            break
        elif user_answer.lower() in ['да', 'yes']:
            main_process()
        else:
            return choose_another_book()

def text_or_audio(book_info, content):
    """"Ask user whether he/she wants to save text or audio file"""
    user_input = input('Save content to a text or audio file? Input text/audio: ')
    if user_input.lower() == 'текст' or user_input.lower() == 'text':
        make_text(book_info[0], content)
    elif user_input.lower() == 'аудио' or user_input.lower() == 'audio':
        make_audio(book_info[0], content)
    else:
        print("Didn't get it...")
        text_or_audio(book_info, content)
    choose_another_book()

def main_process():
    get_titles()
    book_info = user_choise()
    text_parser(book_info)
    content = text_parser(book_info)
    text_or_audio(book_info,content)

#************************************** starting the programm **********************************************************

site_url = 'https://www.booksummary.net/summaries/'
r = requests.get(site_url)
soup = bs(r.content, "lxml")
numbers_of_books = len(soup.findAll('a', href=True, title=True))
conn = create_connection('books.db')
cursor = conn.cursor()

try:
    is_books_table_empty = cursor.execute("""SELECT COUNT(*) FROM books; """).fetchall()
    if is_books_table_empty[0][0] == numbers_of_books:
        print('Ready to work')
        main_process()
    elif is_books_table_empty[0][0] < numbers_of_books:
        print('Table is not full. Starting to add data into the table...')
        title_url_author_parser()
        main_process()
except:
    print("Books table doesn't exist")
    execute_query(conn, create_books_table)
    title_url_author_parser()
    main_process()

conn.close()


import os
from langchain.chat_models import ChatOpenAI
from langchain import PromptTemplate, LLMChain
from langchain.prompts.chat import (
    ChatPromptTemplate,
    SystemMessagePromptTemplate,
    AIMessagePromptTemplate,
    HumanMessagePromptTemplate,
)
from langchain.schema import (
    AIMessage,
    HumanMessage,
    SystemMessage
)

def cl_generator(resume, job_desc, openai_key, temp):
    # Initialize the ChatOpenAI model for generating text
    chat = ChatOpenAI(temperature=temp, openai_api_key=openai_key)

    # Set up the template for the system message
    system_message_template = "You are a student that has to write a cover letter to apply for a job. "
    system_message_prompt = SystemMessagePromptTemplate.from_template(system_message_template)

    # Set up the template for the human message
    human_template = "This is your resume:\n{resume}\nThis is the job description:\n{job_desc}"
    human_message_prompt = HumanMessagePromptTemplate.from_template(human_template)

    # Set up the chat prompt template with system and human messages
    chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])

    # Create the LLMChain with the ChatOpenAI model and chat prompt
    chain = LLMChain(llm=chat, prompt=chat_prompt)

    # Provide the resume and job description as inputs to the chat
    input_data = {"resume": resume, "job_desc": job_desc}

    # Generate the cover letter by running the LLMChain
    return chain.run(input_data)


import datetime
import json
from aiohttp import web
from aiohttp.web import Response
from .spider import get_grade
from .decorator import require_info_login
from .mongo_init import mongo_collection
from jsoncensor import JsonCensor

api = web.Application()

def parseQuertString(query_string):
    if query_string:
        keys = []
        vals = []
        for _ in query_string.split('&'):
            keys.append(_.split('=')[0])
            vals.append(_.split('=')[1])
        args = dict(zip(keys, vals))
        xnm = args.get('xnm')  # xnm: 学年 2016,2017...
        xqm = args.get('xqm')  # xqm:第一学期3，第二学期12,第三学期16
        return xnm, xqm
    return "",""

# ====== async view handlers ======
# 有能从学校处查询的成绩一定使用新成绩
# 只有在从学校那里拿不到的时候才使用缓存的
@require_info_login
async def grade_all_api(request, s, sid, ip):
    current_date = datetime.datetime.now().date()
    current_year = current_date.year
    query_string = request.rel_url.query_string
    xnm,xqm=parseQuertString(query_string)
    key = sid + "_" + xnm + "_" + xqm  # 存入mongodb中的键
    # 查本学年成绩
    if (int(xnm) == current_year and xqm == "3" and datetime.date(current_year, 11, 1) < current_date <= datetime.date(current_year, 12, 30)) \
            or (int(xnm) + 1 == current_year and xqm == "3" and datetime.date(current_year, 1, 1) < current_date <= datetime.date(current_year, 3, 1)) \
            or (int(xnm) + 1 == current_year and xqm != "3" and datetime.date(current_year, 5, 1) < current_date <= datetime.date(current_year, 9, 1)):
        gradeList = await get_grade(s, sid, ip, xnm, xqm)
        val = mongo_collection.find_one({"key": key})
        if gradeList:
            return web.json_response(gradeList)
        elif val is not None:
            # 没有爬到数据,并且缓存中有数据，则返回缓存中的数据
            return web.json_response(json.loads(val))
    else:
        #查以往学年成绩
        gradeList = await get_grade(s, sid, ip, xnm, xqm)
        if gradeList:
            mongo_collection.insert_one({"key": key, "val": json.dumps(gradeList)})
            return web.json_response(gradeList)
        else:
            return Response(body=b'', content_type='application/json', status=404)

async def grade_all_api_cache(request):
    headers = request.headers
    req_headers = dict(headers)
    sid = req_headers.get("Sid")
    query_string = request.rel_url.query_string
    xnm, xqm = parseQuertString(query_string)
    key = sid + "_" + xnm + "_" + xqm  # 存入mongodb中的键
    val = mongo_collection.find_one({"key": key})
    if val is not None:
        val.pop('_id')
        return web.json_response(val)
    return Response(body=b'', content_type='application/json', status=404)

# ios cache
async def grade_ios_client_cache(request):
    headers = request.headers
    req_headers = dict(headers)
    sid = req_headers.get("Sid")
    query_string = request.rel_url.query_string
    xnm, xqm = parseQuertString(query_string)

    print("xnm:", xnm, "xqm:", xqm, "sid:", sid)

    key = sid + "_" + xnm + "_" + xqm
    json_data = await request.json()
    cache_string = json_data.get('data') or None
    if cache_string == None:
        return Response(body=b'{"msg":"need json key:data"}', content_type='application/json', status=400)
    mongo_collection.find_one_and_replace({"key": key}, {"key":key,"val":cache_string}, upsert = True)
    return Response(body=b'', content_type='application/json', status=201)

# ====== url --------- maps  ======
api.router.add_route('GET', '/grade/', grade_all_api, name='grade_all_api')
api.router.add_route('GET','/grade/cache/',grade_all_api_cache,name='grade_all_api_cache')
api.router.add_route('POST', '/grade/cache/', grade_ios_client_cache, name='grade_ios_client_cache')
# ==============================


#!/usr/bin/env python3
"""
Function named index_range that takes two integer
arguments page and page_size and returns a tuple
of size two containing a start index and an end
index corresponding to the range of indexes to
return in a list for those particular pagination
parameters
"""
from typing import Tuple

def index_range(page: int, page_size: int) -> Tuple[int, int]:
    """
    Function that returns a tuple containing a start
    index and an end index
    """
    upper_limit = (page * page_size) - page_size
    lower_limit = page * page_size
    indexes_range = (upper_limit, lower_limit)
    return indexes_range


import tensorflow as tf

from transformer.CustomSchedule import CustomSchedule
from transformer.LoadDebates import load_conversations
from transformer.MultiHeadAttention import MultiHeadAttention
from transformer.PositionalEncoding import PositionalEncoding
from parseDebate.Debates import Debates

assert tf.__version__.startswith('2')
tf.random.set_seed(1234)

import tensorflow_datasets as tfds

import os
import re
import numpy as np

import matplotlib.pyplot as plt

# Maximum sentence length
MAX_LENGTH = 4000

class Tokenizer:
    def __init__(self, questions, answers):
        self.tokenizer = tfds.features.text.SubwordTextEncoder.build_from_corpus(
            questions + answers, target_vocab_size=2 ** 13)

        # Define start and end token to indicate the start and end of a sentence
        self.START_TOKEN, self.END_TOKEN = [self.tokenizer.vocab_size], [self.tokenizer.vocab_size + 1]

        # Vocabulary size plus start and end token
        self.VOCAB_SIZE = self.tokenizer.vocab_size + 2

    def tokenize_and_filter(self, inputs, outputs):
        tokenized_inputs, tokenized_outputs = [], []

        for (sentence1, sentence2) in zip(inputs, outputs):
            # tokenize sentence
            sentence1 = self.START_TOKEN + self.tokenizer.encode(sentence1) + self.END_TOKEN
            sentence2 = self.START_TOKEN + self.tokenizer.encode(sentence2) + self.END_TOKEN
            # check tokenized sentence max length
            if len(sentence1) <= MAX_LENGTH and len(sentence2) <= MAX_LENGTH:
                tokenized_inputs.append(sentence1)
                tokenized_outputs.append(sentence2)

        # pad tokenized sentences
        tokenized_inputs = tf.keras.preprocessing.sequence.pad_sequences(
            tokenized_inputs, maxlen=MAX_LENGTH, padding='post')
        tokenized_outputs = tf.keras.preprocessing.sequence.pad_sequences(
            tokenized_outputs, maxlen=MAX_LENGTH, padding='post')

        return tokenized_inputs, tokenized_outputs

def create_padding_mask(x):
  mask = tf.cast(tf.math.equal(x, 0), tf.float32)
  # (batch_size, 1, 1, sequence length)
  return mask[:, tf.newaxis, tf.newaxis, :]

def create_look_ahead_mask(x):
  seq_len = tf.shape(x)[1]
  look_ahead_mask = 1 - tf.linalg.band_part(tf.ones((seq_len, seq_len)), -1, 0)
  padding_mask = create_padding_mask(x)
  return tf.maximum(look_ahead_mask, padding_mask)

def encoder_layer(units, d_model, num_heads, dropout, name="encoder_layer"):
  inputs = tf.keras.Input(shape=(None, d_model), name="inputs")
  padding_mask = tf.keras.Input(shape=(1, 1, None), name="padding_mask")

  attention = MultiHeadAttention(
      d_model, num_heads, name="attention")({
          'query': inputs,
          'key': inputs,
          'value': inputs,
          'mask': padding_mask
      })
  attention = tf.keras.layers.Dropout(rate=dropout)(attention)
  attention = tf.keras.layers.LayerNormalization(
      epsilon=1e-6)(inputs + attention)

  outputs = tf.keras.layers.Dense(units=units, activation='relu')(attention)
  outputs = tf.keras.layers.Dense(units=d_model)(outputs)
  outputs = tf.keras.layers.Dropout(rate=dropout)(outputs)
  outputs = tf.keras.layers.LayerNormalization(
      epsilon=1e-6)(attention + outputs)

  return tf.keras.Model(
      inputs=[inputs, padding_mask], outputs=outputs, name=name)

def encoder(vocab_size,
            num_layers,
            units,
            d_model,
            num_heads,
            dropout,
            name="encoder"):
  inputs = tf.keras.Input(shape=(None,), name="inputs")
  padding_mask = tf.keras.Input(shape=(1, 1, None), name="padding_mask")

  embeddings = tf.keras.layers.Embedding(vocab_size, d_model)(inputs)
  embeddings *= tf.math.sqrt(tf.cast(d_model, tf.float32))
  embeddings = PositionalEncoding(vocab_size, d_model)(embeddings)

  outputs = tf.keras.layers.Dropout(rate=dropout)(embeddings)

  for i in range(num_layers):
    outputs = encoder_layer(
        units=units,
        d_model=d_model,
        num_heads=num_heads,
        dropout=dropout,
        name="encoder_layer_{}".format(i),
    )([outputs, padding_mask])

  return tf.keras.Model(
      inputs=[inputs, padding_mask], outputs=outputs, name=name)

def decoder_layer(units, d_model, num_heads, dropout, name="decoder_layer"):
    inputs = tf.keras.Input(shape=(None, d_model), name="inputs")
    enc_outputs = tf.keras.Input(shape=(None, d_model), name="encoder_outputs")
    look_ahead_mask = tf.keras.Input(
        shape=(1, None, None), name="look_ahead_mask")
    padding_mask = tf.keras.Input(shape=(1, 1, None), name='padding_mask')

    attention1 = MultiHeadAttention(
        d_model, num_heads, name="attention_1")(inputs={
        'query': inputs,
        'key': inputs,
        'value': inputs,
        'mask': look_ahead_mask
    })
    attention1 = tf.keras.layers.LayerNormalization(
        epsilon=1e-6)(attention1 + inputs)

    attention2 = MultiHeadAttention(
        d_model, num_heads, name="attention_2")(inputs={
        'query': attention1,
        'key': enc_outputs,
        'value': enc_outputs,
        'mask': padding_mask
    })
    attention2 = tf.keras.layers.Dropout(rate=dropout)(attention2)
    attention2 = tf.keras.layers.LayerNormalization(
        epsilon=1e-6)(attention2 + attention1)

    outputs = tf.keras.layers.Dense(units=units, activation='relu')(attention2)
    outputs = tf.keras.layers.Dense(units=d_model)(outputs)
    outputs = tf.keras.layers.Dropout(rate=dropout)(outputs)
    outputs = tf.keras.layers.LayerNormalization(
        epsilon=1e-6)(outputs + attention2)

    return tf.keras.Model(
        inputs=[inputs, enc_outputs, look_ahead_mask, padding_mask],
        outputs=outputs,
        name=name)

def decoder(vocab_size,
            num_layers,
            units,
            d_model,
            num_heads,
            dropout,
            name='decoder'):
    inputs = tf.keras.Input(shape=(None,), name='inputs')
    enc_outputs = tf.keras.Input(shape=(None, d_model), name='encoder_outputs')
    look_ahead_mask = tf.keras.Input(
        shape=(1, None, None), name='look_ahead_mask')
    padding_mask = tf.keras.Input(shape=(1, 1, None), name='padding_mask')

    embeddings = tf.keras.layers.Embedding(vocab_size, d_model)(inputs)
    embeddings *= tf.math.sqrt(tf.cast(d_model, tf.float32))
    embeddings = PositionalEncoding(vocab_size, d_model)(embeddings)

    outputs = tf.keras.layers.Dropout(rate=dropout)(embeddings)

    for i in range(num_layers):
        outputs = decoder_layer(
            units=units,
            d_model=d_model,
            num_heads=num_heads,
            dropout=dropout,
            name='decoder_layer_{}'.format(i),
        )(inputs=[outputs, enc_outputs, look_ahead_mask, padding_mask])

    return tf.keras.Model(
        inputs=[inputs, enc_outputs, look_ahead_mask, padding_mask],
        outputs=outputs,
        name=name)

def transformer(vocab_size,
                num_layers,
                units,
                d_model,
                num_heads,
                dropout,
                name="transformer"):
  inputs = tf.keras.Input(shape=(None,), name="inputs")
  dec_inputs = tf.keras.Input(shape=(None,), name="dec_inputs")

  enc_padding_mask = tf.keras.layers.Lambda(
      create_padding_mask, output_shape=(1, 1, None),
      name='enc_padding_mask')(inputs)
  # mask the future tokens for decoder inputs at the 1st attention block
  look_ahead_mask = tf.keras.layers.Lambda(
      create_look_ahead_mask,
      output_shape=(1, None, None),
      name='look_ahead_mask')(dec_inputs)
  # mask the encoder outputs for the 2nd attention block
  dec_padding_mask = tf.keras.layers.Lambda(
      create_padding_mask, output_shape=(1, 1, None),
      name='dec_padding_mask')(inputs)

  enc_outputs = encoder(
      vocab_size=vocab_size,
      num_layers=num_layers,
      units=units,
      d_model=d_model,
      num_heads=num_heads,
      dropout=dropout,
  )(inputs=[inputs, enc_padding_mask])

  dec_outputs = decoder(
      vocab_size=vocab_size,
      num_layers=num_layers,
      units=units,
      d_model=d_model,
      num_heads=num_heads,
      dropout=dropout,
  )(inputs=[dec_inputs, enc_outputs, look_ahead_mask, dec_padding_mask])

  outputs = tf.keras.layers.Dense(units=vocab_size, name="outputs")(dec_outputs)

  return tf.keras.Model(inputs=[inputs, dec_inputs], outputs=outputs, name=name)

def loss_function(y_true, y_pred):
    y_true = tf.reshape(y_true, shape=(-1, MAX_LENGTH - 1))

    loss = tf.keras.losses.SparseCategoricalCrossentropy(
        from_logits=True, reduction='none')(y_true, y_pred)

    mask = tf.cast(tf.not_equal(y_true, 0), tf.float32)
    loss = tf.multiply(loss, mask)

    return tf.reduce_mean(loss)

if __name__ == '__main__':
    debate_holder = Debates()

    debate_holder.between('2020-06-24', '2020-06-24')
    x = debate_holder.debates_from(25353)

    [questions, answers] = load_conversations(x)

    tokenizer = Tokenizer(questions, answers)

    questions, answers = tokenizer.tokenize_and_filter(questions, answers)
    BATCH_SIZE = 64
    BUFFER_SIZE = 20000

    # decoder inputs use the previous target as input
    # remove START_TOKEN from targets
    dataset = tf.data.Dataset.from_tensor_slices((
        {
            'inputs': questions,
            'dec_inputs': answers[:, :-1]
        },
        {
            'outputs': answers[:, 1:]
        },
    ))

    dataset = dataset.cache()
    dataset = dataset.shuffle(BUFFER_SIZE)
    dataset = dataset.batch(BATCH_SIZE)
    dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)

    # Hyper-parameters
    NUM_LAYERS = 8
    D_MODEL = 256
    NUM_HEADS = 8
    UNITS = 512
    DROPOUT = 0.1

    model = transformer(
        vocab_size=tokenizer.VOCAB_SIZE,
        num_layers=NUM_LAYERS,
        units=UNITS,
        d_model=D_MODEL,
        num_heads=NUM_HEADS,
        dropout=DROPOUT)

    learning_rate = CustomSchedule(D_MODEL)

    optimizer = tf.keras.optimizers.Adam(
        learning_rate, beta_1=0.9, beta_2=0.98, epsilon=1e-9)

    def accuracy(y_true, y_pred):
        # ensure labels have shape (batch_size, MAX_LENGTH - 1)
        y_true = tf.reshape(y_true, shape=(-1, MAX_LENGTH - 1))
        return tf.keras.metrics.sparse_categorical_accuracy(y_true, y_pred)

    model.compile(optimizer=optimizer, loss=loss_function, metrics=[accuracy])

    EPOCHS = 2

    model.fit(dataset, epochs=EPOCHS)


from question_model import Question
from data import question_data
from quiz_brain import QuizBrain

question_bank = list()

for item in question_data:
    q_text = item["text"]
    q_answer = item["answer"]
    new_q = Question(q_text, q_answer)
    question_bank.append(new_q)

quiz = QuizBrain(question_bank)

while quiz.has_questions():
    quiz.next_question()
quiz.quiz_complete()


import sys

from cv2 import cv2

if __name__ == '__main__':
    if len(sys.argv) < 2:
        print("set file name")
        sys.exit(1)
    name = sys.argv[1]
    img = cv2.imread(name)
    cv2.imshow('my_image', img)
    cv2.waitKey(5000)
    cv2.destroyAllWindows()