from __future__ import absolute_import
from __future__ import print_function
import ast
import json
import re
import sys
from xml.etree import ElementTree

import requests
from bs4 import BeautifulSoup

OLD_MLB_PATTERNS = [
    '(?P<domain>mi?lb).com/(?:\w{2,3}/)?video/(?:topic/\d+/)?v(?P<content_id>\d+)',
    '(?P<domain>mi?lb).com/.*content_id=(?P<content_id>\d+)',
    '(?P<domain>mi?lb).com/.*/v(?P<content_id>\d+)',
    '(?P<domain>mi?lb).com/.*/c-(?P<content_id>\d+)',
]

MLB_PATTERNS = [
    '\S*mi?lb.com/.*video/(?P<content_id>[\w\-]*)'
]

MLB_SKIP_PATTERNS = [
    'mlb.com/news'
]

SHORTURL_PATTERNS = [
    '(?:https?://)?(?P<url>atmlb.com\S+)'
]

MLB_VIDEO_JSON_FORMAT = 'https://www.mlb.com/data-service/en/videos/{content_id}'
# Used to find true URL instead of inferring date
MLB_OLD_VIDEO_XML_FORMAT = 'http://www.{domain}.com/gen/multimedia/detail/{first}/{second}/{third}/{content_id}.xml'

MLB_XML_IGNORED_SUBJECTS = [
    'MLBCOM_CONDENSED_GAME',
    'LIVE_EVENT_COVERAGE'
]

def skip_match(text):
    for skip_pattern in MLB_SKIP_PATTERNS:
        skip_match = re.search(skip_pattern, text)
        if skip_match:
            return True
    return False

def find_mlb_links(text):
    shorturl_text = ''
    for shorturl_pattern in SHORTURL_PATTERNS:
        matches = re.finditer(shorturl_pattern, text)
        for match in matches:
            shorturl_headers = requests.head('http://{}'.format(match.group('url')))
            if shorturl_headers.status_code == 301:
                shorturl_text += '{} '.format(shorturl_headers.headers['location'])

    text += ' {}'.format(shorturl_text)

    old_re_matches = []

    for mlb_pattern in OLD_MLB_PATTERNS:
        matches = re.finditer(mlb_pattern, text)
        for match in matches:
            if skip_match(match.string):
                continue
            print("    old match {} found: {}".format(match.groupdict(), text))
            old_re_matches.append(match)

    old_matches, old_matched_content_ids = format_old_comments(old_re_matches)

    re_matches = []

    for mlb_pattern in MLB_PATTERNS:
        matches = re.finditer(mlb_pattern, text)
        for match in matches:
            # need to not duplicate old links
            if match.group() in old_matched_content_ids:
                break
            else:
                print("    match {} found: {}".format(match.group(), text))
                re_matches.append(match)

    return old_matches + format_comments(re_matches)

def format_comments(regex_matches):
    unique_content_id = set()
    formatted_comments = []

    for match in regex_matches:
        if match.group('content_id') in unique_content_id:
            continue
        unique_content_id.add(match.group('content_id'))
        media_links = get_media_for_content_id(match)
        if not media_links:
            print("    no media link found for {}".format(match.group('content_id')))
            continue
        formatted_comments.append(format_link(media_links))

    return formatted_comments

def get_media_for_content_id(match):

    def get_media_from_json(match_):
        content_id = match_.group('content_id')
        url = MLB_VIDEO_JSON_FORMAT.format(**{
            "content_id": content_id
        })
        video_data = json.loads(requests.get(url).content)
        if 'playbacks' not in video_data:
            return {}

        media = []

        for playback in video_data['playbacks']:
            if not playback['url'].endswith('mp4'):
                continue
            media.append((playback['url'], playback['name']))

        return {'title': video_data['blurb'], 'media': media}

    def get_media_from_html(match_):
        def search_ld_json_block(ld_json_block):
            video_info = json.loads(ld_json_block.encode_contents())

            title = video_info['name']

            # MLB now providing streamable links, which is nice.
            media = []
            if video_info['embedUrl']:
                media.append((video_info['embedUrl'], "Streamable Link"))
            if video_info['contentUrl'] and not video_info['contentUrl'].endswith('m3u8'):
                media.append((video_info['contentUrl'], "MP4 Link"))

            return {'title': title, 'media': media}

        def search_main_block(main_script_contents):
            media = []
            title = ""
            for line in main_script_contents.string.splitlines():
                init_state = line.split("__VIDEO_INIT_STATE__ = ")
                if len(init_state) < 2:
                    continue
                init_state_dict = json.loads(ast.literal_eval(init_state[1].rstrip(";")))
                for key in init_state_dict.keys():
                    if not key.startswith("MediaPlayback:"):
                        continue
                    media_playback = init_state_dict[key]
                    if 'blurb' in media_playback:
                        title = media_playback['blurb']
                    for feed in media_playback.get('feeds', []):
                        for playback in feed.get('playbacks', []):
                            if 'url' not in playback or 'name' not in playback or not playback['url'].endswith('mp4'):
                                continue
                            media.append((playback['url'], playback['name']))

            return {'title': title, 'media': media}

        headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) '
            'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
        }
        resp = requests.get(parse_reddit_formatted_link(match_.group()), headers=headers)
        page_content = BeautifulSoup(resp.content, 'html.parser')

        ld_json_script_contents = page_content.find("script", type="application/ld+json")
        ld_json_results = search_ld_json_block(ld_json_script_contents)

        main_script_contents = page_content.find("main").find("script")
        main_script_results = search_main_block(main_script_contents)

        return {'title': ld_json_results['title'],
                'media': ld_json_results['media'] + main_script_results['media']}

    json_media = get_media_from_json(match)
    html_media = get_media_from_html(match)

    unique_links = set()

    # dedupe across json/html
    media = []
    for media_link, link_text in json_media['media'] + html_media['media']:
        if media_link in unique_links:
            continue
        unique_links.add(media_link)
        media.append((media_link, link_text))

    return {'title': html_media['title'], 'media': media}

# Reddit links can either be linked directly
# - https://www.mlb.com/video/edwin-rios-called-out-on-strikes
# Or they can be formatted
# - [https://www.mlb.com/video/encarnacion-gets-yanks-on-board](https://www.mlb.com/video/encarnacion-gets-yanks-on-board)
# This method extracts the link from the second format, or returns if unformatted
def parse_reddit_formatted_link(link):
    return link.split('](').pop().rstrip(')')

# Expects media_links with keys:
# - title: string
# - media: list of tuples of (link, text for link)
def format_link(media_links):
    title = media_links['title']

    streamable_links = []
    mp4_links = []
    mp4_sorter = []

    for media_link, link_text in media_links['media']:
        # Hack to skip getting content size if it's on Streamable
        if link_text == "Streamable Link":
            streamable_links.append("[{}]({})".format(link_text, media_link))
            continue

        headers = requests.head(media_link)
        # Can be a redirect if it's a fastball-clips URL
        if headers.status_code == 200:
            size_mb = round(float(headers.headers['content-length']) / (1024 ** 2), 2)
            mp4_sorter.append((link_text, media_link, size_mb))

    # Sort links by descending size
    mp4_sorter.sort(key=lambda x: x[2], reverse=True)
    for link_text, media_link, size_mb in mp4_sorter:
        # reformat the link text to be more human readable
        if link_text == 'highBit':
            link_text = 'High Definition'
        elif link_text == 'mp4Avc':
            link_text = 'Standard Definiton'
        else:
            # Look for things like FLASH_1800K_960x540
            resolution_match = re.match('\w+_(?P<resolution>\d+K)_[\dX]+', link_text)
            if resolution_match:
                link_text = resolution_match.group('resolution')

        mp4_links.append("[{}]({}) ({} MB)".format(link_text, media_link, size_mb))

    return ["Video: {}".format(title)] + streamable_links + mp4_links + ["___________"]

def format_old_comments(regex_matches):
    unique_content_id = set()
    formatted_comments = []
    matched_content_ids = set()

    for match in regex_matches:
        if match.group('content_id') in unique_content_id:
            continue
        unique_content_id.add(match.group('content_id'))
        media_links = get_media_for_old_content_id(match)
        if not media_links:
            print("    no media link found for {}".format(match.group('content_id')))
            continue
        formatted_comments.append(format_link(media_links))
        matched_content_ids.add(match.group('content_id'))

    return formatted_comments, matched_content_ids

def get_media_for_old_content_id(match):
    content_id = match.group('content_id')
    url = MLB_OLD_VIDEO_XML_FORMAT.format(**{
        "domain": match.group('domain'),
        "first": content_id[-3],
        "second": content_id[-2],
        "third": content_id[-1],
        "content_id": content_id
    })
    try:
        tree = ElementTree.fromstring(requests.get(url).content)
    except Exception as e:
        print("    error parsing/receiving XML from url {}: {}".format(url, e))
        return {}

    keyword = tree.find('keywords').find('keyword[@type="subject"]')
    if keyword.get('value') in MLB_XML_IGNORED_SUBJECTS:
        return {}

    title = tree.find('blurb').text
    media_tags = tree.findall('url[@playback_scenario]')

    small_mp4_size = 0
    small_mp4_threshold = 640
    small_mp4_url = None

    largest_mp4_size = 0
    largest_mp4_url = None

    for media_tag in media_tags:
        if not media_tag.text:
            continue
        mp4_size_match = re.search('_(?P<mp4_size>\d+)K\.mp4', media_tag.text)
        if mp4_size_match is not None:
            vid_url = media_tag.text.replace('http:', 'https:')
            mp4_size = int(mp4_size_match.group('mp4_size'))
            if mp4_size > largest_mp4_size:
                largest_mp4_size = mp4_size
                largest_mp4_url = vid_url
            if small_mp4_threshold > mp4_size > small_mp4_size:
                small_mp4_size = mp4_size
                small_mp4_url = vid_url

    # Need to match at least one else return nothing
    if largest_mp4_url is None:
        return {}

    if small_mp4_size == largest_mp4_size or small_mp4_url is None:
        largest_mp4_text = "MP4 Video"
    else:
        largest_mp4_text = "Larger Version"

    media = {"title": title, "media": [(largest_mp4_url, largest_mp4_text)]}
    if small_mp4_size != largest_mp4_size and small_mp4_url is not None:
        media['media'].append((small_mp4_url, 'Smaller Version'))

    return media

# Pass a test URL as the first arg to do a test run
if __name__ == '__main__':
    print(find_mlb_links(sys.argv[1]))


#!/usr/bin/python
#==============================================================
# Implicit integrator schemes for stiff coupled ODEs.
# Containing the functions
#
#==============================================================
# required libraries
import argparse	                 # allows us to deal with arguments to main()
from argparse import RawTextHelpFormatter
import numpy as np               # numerical routines (arrays, math functions etc)
import matplotlib.pyplot as plt  # plotting commands
import p358utilities as util     # for rescaleplot

import ode_integrators as odeint # contains the drivers for ODE integration
import ode_step as step          # the stepper functions

#==============================================================
# function dydx = get_dydx(x,y,dx)
#
# Calculates RHS for stiff test equation
#--------------------------------------------------------------

def get_dydx(x,y,dx):
    dydx    = np.zeros(2)
    dydx[0] =  998.0*y[0]+1998.0*y[1]
    dydx[1] = -999.0*y[0]-1999.0*y[1]
    return dydx

#==============================================================
#==============================================================
# main
#
# parameters:
#   stepper: ODE solver (see help function below)
#

def main():

    parser = argparse.ArgumentParser(formatter_class=RawTextHelpFormatter)
    parser.add_argument("stepper",type=str,default='euler',
                        help="stepping function:\n"
                             "   euler    : Euler step\n"
                             "   rk2      : Runge-Kutta 2nd order\n"
                             "   rk4      : Runge-Kutta 4th order\n"
                             "   backeuler: implicit Euler step\n")
    args   = parser.parse_args()
    if (args.stepper == "euler"):
        fORD = step.euler
    elif (args.stepper == "rk2"):
        fORD = step.rk2
    elif (args.stepper == "rk4"):
        fORD = step.rk4
    elif (args.stepper == "backeuler"):
        fORD = step.backeuler
    else:
        raise Exception("invalid stepper %s" % (args.stepper))

    # initialization
    nstep    = 100
    x0       = 0.0
    x1       = 1.0
    y0       = np.array([1.0,0.0])
    fINT     = odeint.ode_ivp
    fRHS     = get_dydx
    fBVP     = 0

    # solve
    x,y,it   = fINT(fRHS,fORD,fBVP,x0,y0,x1,nstep)

    # Your code should show three plots in one window:
    # (1) the dependent variables y[0,:] and y[1,:] against x
    # (2) the residual y[0,:]-u and y[1,:]-v against x, see homework sheet
    # (3) the number of iterations  against x.
    # from here ??????
    xmin = np.min(x)
    xmax = np.max(x)
    ymin = np.min(y)
    ymax = np.max(y)
    ftsz = 10
    fig  = plt.figure(num=1,figsize=(6,8),dpi=100,facecolor='white')
    u    = 2.0*np.exp(-x)-np.exp(-1e3*x)
    v    = -np.exp(-x)+np.exp(-1e3*x)
    diff = np.zeros((2,x.size))
    diff[0,:] = y[0]-u
    diff[1,:] = y[1]-v

    plt.subplot(311)
    plt.plot(x,y[0,:],linestyle='-',color='blue',linewidth=1.0,label='u')
    plt.plot(x,u,linestyle='--',color='blue',linewidth=1.0,label='u [analytic]')
    plt.plot(x,y[1,:],linestyle='-',color='red',linewidth=1.0,label='v')
    plt.plot(x,v,linestyle='--',color='red',linewidth=1.0,label='v [analytic]')
    plt.xlabel('x',fontsize=ftsz)
    plt.ylabel('y',fontsize=ftsz)
    plt.legend(fontsize=ftsz)
    util.rescaleplot(x,y,plt,0.05)
    plt.tick_params(labelsize=ftsz)
    plt.subplot(312)
    plt.plot(x,diff[0,:],linestyle='-',color='blue',linewidth=1.0,label='residual u')
    plt.plot(x,diff[1,:],linestyle='-',color='red',linewidth=1.0,label='residual v')
    plt.xlabel('x',fontsize=ftsz)
    plt.ylabel('y-y[analytic]',fontsize=ftsz)
    plt.legend(fontsize=ftsz)
    util.rescaleplot(x,diff,plt,0.05)
    plt.tick_params(labelsize=ftsz)
    plt.subplot(313)
    plt.plot(x[1:x.size],np.log10(it[1:x.size]),linestyle='-',color='black',linewidth=1.0,label='iterations')
    plt.xlabel('x',fontsize=ftsz)
    plt.ylabel('log #',fontsize=ftsz)
    plt.legend(fontsize=ftsz)
    util.rescaleplot(x[1:x.size],np.log10(it[1:x.size]),plt,0.05)
    plt.tick_params(labelsize=ftsz)

    plt.show()
    # to here ??????

#==============================================================

main()


from web.edit.edit_common import showdir
from core.users import getHomeDir
from core.transition import current_user
import logging
from contenttypes import Collections, Container
from core import Node
from core import db
from core.translation import lang as _lang
from core.translation import t as _t

logg = logging.getLogger(__name__)
q = db.query

def getInformation():
    return {"version":"1.1", "system":1}

def getContent(req, ids):
    logg.error("publish.getContent")
    user = current_user
    publishdir = q(Node).get(ids[0])
    ret = ""

    errorids = []
    publisherror = []
    changes = []
    if "dopublish" in req.params.keys():
        logg.debug("dopublish")
        num_req_err_nodes = 0
        num_db_err_nodes = 0
        num_rights_err_nodes = 0
        objlist = []
        for key in req.params.keys():
            if key.isdigit():
                objlist.append(key)
                src = q(Node).get(req.params.get("id"))

        for obj_id in objlist:
            remove_from_src = False
            obj = q(Node).get(obj_id)
            metadatatype = obj.metadatatype
            mask_validated = False
            for mask in metadatatype.getMasks(type="edit"): # check required fields
                if mask.has_read_access() and mask.getName() == obj.system_attrs.get("edit.lastmask"):
                    for f in mask.validateNodelist([obj]):
                        errorids.append(f)
                    mask_validated = True

            logg.error(errorids)
            if len(errorids)>0: # object faulty
                num_req_err_nodes +=1
                # if object faulty, it is not necessary to do the rest of error handling for this object
                continue

            if not mask_validated:
                msg = "user %r going to publish node %r without having validated edit.lastmask" % (user, obj)
                logg.warning(msg)
                # should we validate standard edit mask here?

            for dest_id in req.params.get("destination", "").split(","):
                if not dest_id: # no destination given
                    continue

                dest = q(Node).get(dest_id)

                # XXX: this error handling should be revised, I think...

                error = False # general rights error
                if not src.has_read_access():
                    logg.error("Error in publishing of node %r: source position %r has no read access.", obj.id, src.id)
                    error = True
                if not dest.has_write_access():
                    logg.error("Error in publishing of node %r: destination %r has no write access.", obj.id, dest.id)
                    error = True
                if not obj.has_write_access():
                    logg.error("Error in publishing of node %r: object has no write access.", obj.id)
                    error = True
                if not isinstance(dest, Container) or dest == src:
                    # cannot happen normally due to selection tree
                    logg.error("Error in publishing of node %r: destination %r is not a directory.", obj.id, dest.id)
                    error = True

                if not error:
                    if not obj.is_descendant_of(dest):
                        dest.children.append(obj)
                        remove_from_src = True

                        if dest.id not in changes:
                            changes.append(dest.id)
                        if src.id not in changes:
                            changes.append(src.id)
                        logg.info("%s published %s (%s, %s) from src %s (%s, %s) to dest %s (%s, %s)", user.login_name,
                                  obj.id, obj.name, obj.type,
                                  src.id, src.name, src.type,
                                  dest.id, dest.name, dest.type)
                    else:
                        num_rights_err_nodes += 1
                        errorids.append(obj.id)
                        logg.error("Error in publishing of node {}: Destination node {} is child of node.".format(obj_id, dest.id))
                else:
                    # error already logged
                    num_rights_err_nodes += 1
                    errorids.append(obj.id)

            if remove_from_src:
                try:
                    src.children.remove(obj)
                    db.session.commit()
                except:
                    num_db_err_nodes += 1
                    logg.exception("Error in publishing of node {}: Database error".format(obj.id))
                    errorids.append(obj.id)

        # error messages for publishing assistant
        if num_req_err_nodes > 0:
            if num_req_err_nodes < 2:
                publisherror.append(_t(_lang(req), "error_publish_single_node"))
            else:
                publisherror.append(_t(_lang(req), "error_publish_multiple_nodes"))
        if num_rights_err_nodes > 0:
            if num_rights_err_nodes < 2:
                publisherror.append(_t(_lang(req), "error_publish_rights_single"))
            else:
                publisherror.append(_t(_lang(req), "error_publish_rights_multiple"))
        if num_db_err_nodes > 0:
            if num_db_err_nodes < 2:
                publisherror.append(_t(_lang(req), "error_publish_database_single"))
            else:
                publisherror.append(_t(_lang(req), "error_publish_database_multiple"))

        v = {}
        v["id"] = publishdir.id
        v["change"] = changes
        ret += req.getTAL("web/edit/modules/publish.html", v, macro="reload")

    # build normal window
    stddir = ""  # preset value for destination ids
    stdname = ""

    v = {"id": publishdir.id,
         "stddir": stddir,
         "stdname": stdname,
         "showdir": showdir(req,
                            publishdir,
                            publishwarn=None,
                            markunpublished=1,
                            faultyidlist=errorids,
                            ),
         "basedir": q(Collections).one(),
         "script": "var currentitem = '%s';\nvar currentfolder = '%s'" % (publishdir.id, publishdir.id), "idstr": ids,
         "faultyerrlist": publisherror,
        }
    v["csrf"] = req.csrf_token.current_token
    ret += req.getTAL("web/edit/modules/publish.html", v, macro="publish_form")
    return ret


import os

import numpy
import pandas
from torchvision.datasets.utils import download_url

from lib.src.data_utils import DATA_FOLDER
from lib.src.os_utils import safe_makedirs

ADULT_RAW_COL_NAMES = ["age", "workclass", "fnlwgt", "education", "education-num", "marital-status",
                       "occupation", "relationship", "race", "sex", "capital-gain", "capital-loss",
                       "hours-per-week", "native-country", "income", ]
# this are categorical column indices; doesnot include sex and income column (these are binary)
ADULT_RAW_COL_FACTOR = [1, 3, 5, 6, 7, 8, 13]

def maybe_download():
    path = os.path.join(DATA_FOLDER, "adult")
    safe_makedirs(path)
    download_url("http://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data", path)
    download_url("http://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.test", path)

def load_adult(val_size=0.0):
    maybe_download()
    # taken from https://github.com/dcmoyer/inv-rep/blob/master/src/uci_data.py

    # load data as pandas table
    train_data = pandas.read_table(os.path.join(DATA_FOLDER, "adult", "adult.data"), delimiter=", ",
                                   header=None, names=ADULT_RAW_COL_NAMES, na_values="?",
                                   keep_default_na=False)
    test_data = pandas.read_table(os.path.join(DATA_FOLDER, "adult", "adult.test"), delimiter=", ",
                                  header=None, names=ADULT_RAW_COL_NAMES, na_values="?",
                                  keep_default_na=False, skiprows=1)

    # drop missing val
    train_data.dropna(inplace=True)
    test_data.dropna(inplace=True)

    # concatenate and binarize categorical variables
    all_data = pandas.concat([train_data, test_data])
    all_data = pandas.get_dummies(all_data,
                                  columns=[ADULT_RAW_COL_NAMES[i] for i in ADULT_RAW_COL_FACTOR])
    # fix binary variables now
    all_data.loc[all_data.income == ">50K", "income"] = 1
    all_data.loc[all_data.income == ">50K.", "income"] = 1
    all_data.loc[all_data.income == "<=50K", "income"] = 0
    all_data.loc[all_data.income == "<=50K.", "income"] = 0

    all_data.loc[all_data.sex == "Female", "sex"] = 0
    all_data.loc[all_data.sex == "Male", "sex"] = 1
    cutoff = train_data.shape[0]

    all_x = all_data.iloc[:, (all_data.columns != "income") & (all_data.columns != "sex")]
    all_c = all_data.iloc[:, all_data.columns == "sex"]
    all_labels = all_data.iloc[:, all_data.columns == "income"]

    # col_valid = [len(all_x.iloc[:, all_x.columns==x].unique()) > 1 for x in all_x.columns]
    # all_x = all_x.iloc[:, col_valid]

    # normalization
    maxes = all_x.max(axis=0)
    all_x = all_x / maxes

    train_data = all_x[:cutoff]
    train_c = all_c[:cutoff]
    train_labels = all_labels[:cutoff]

    test_data = all_x[cutoff:]
    test_c = all_c[cutoff:]
    test_labels = all_labels[cutoff:]

    # split off validation data (we keep val_size for training 0, so this code is never run)
    if val_size != 0:
        # # shuffle
        # train_data.sample(frac=1, random_state=0)
        # train_c.sample(frac=1, random_state=0)
        # train_labels.sample(frac=1, random_state=0)

        val_cutoff = int((1 - val_size) * train_data.shape[0])

        val_data = train_data.iloc[val_cutoff:, :]
        train_data = train_data.iloc[:val_cutoff, :]

        val_labels = train_labels.iloc[val_cutoff:, :]
        train_labels = train_labels.iloc[:val_cutoff, :]

        val_c = train_c.iloc[val_cutoff:, :]
        train_c = train_c.iloc[:val_cutoff, :]

    return {"train": (
        train_data.to_numpy(),
        train_c.to_numpy(dtype=numpy.int),
        train_labels.to_numpy(dtype=numpy.int),
    ), "valid": None if val_size == 0 else (
        val_data.to_numpy(),
        val_c.to_numpy(dtype=numpy.int),
        val_labels.to_numpy(dtype=numpy.int),
    ), "test": (
        test_data.to_numpy(),
        test_c.to_numpy(dtype=numpy.int),
        test_labels.to_numpy(dtype=numpy.int),
    )}

if __name__ == "__main__":
    data = load_adult(0.2)
    print("Adult dataset:")
    print(f"Train size: {data['train'][0].shape}")
    if data['valid'] is not None:
        print(f"Val size: {data['valid'][0].shape}")
    print(f"Test size: {data['test'][0].shape}")


from math import e
import torch
import torch.nn as nn
from torchcrf import CRF
from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence
from transformers import BertModel, BertTokenizer

class CWS(nn.Module):

    def __init__(self, vocab_size, tag2id, embedding_dim, hidden_dim):
        super(CWS, self).__init__()
        self.embedding_dim = embedding_dim
        self.hidden_dim = hidden_dim
        self.vocab_size = vocab_size
        self.tag2id = tag2id
        self.tagset_size = len(tag2id)

        self.tokenizer = BertTokenizer.from_pretrained("hfl/chinese-roberta-wwm-ext")
        self.word_embeds = BertModel.from_pretrained("hfl/chinese-roberta-wwm-ext")

        self.lstm = nn.LSTM(embedding_dim, hidden_dim // 2, num_layers=1,
                            bidirectional=True, batch_first=True)
        self.hidden2tag = nn.Linear(hidden_dim, self.tagset_size)

        self.crf = CRF(4, batch_first=True)

    def init_hidden(self, batch_size, device):
        return (torch.randn(2, batch_size, self.hidden_dim // 2, device=device),
                torch.randn(2, batch_size, self.hidden_dim // 2, device=device))

    def _get_lstm_features(self, input_ids, input_mask):
        batch_size = input_ids.size(0)
        embeds = self.word_embeds(input_ids=input_ids, attention_mask=input_mask, output_hidden_states=True, output_attentions=True)
        # idx->embedding
        all_hidden_states, all_attention = embeds[-2:]
        embeds = all_hidden_states[-2]

        # LSTM forward
        self.hidden = self.init_hidden(batch_size, input_ids.device)
        lstm_out, self.hidden = self.lstm(embeds, self.hidden)
        lstm_feats = self.hidden2tag(lstm_out)
        return lstm_feats

    def forward(self, input_ids, label, input_mask, output_mask):
        # print(input_ids)
        # print(label)
        # print(input_mask)
        # print(output_mask)
        emissions = self._get_lstm_features(input_ids, input_mask)
        loss = -self.crf(emissions, label, output_mask, reduction='mean')
        return loss

    def infer(self, input_ids, input_mask, output_mask):
        emissions = self._get_lstm_features(input_ids, input_mask)
        return self.crf.decode(emissions, output_mask)


#encoding:utf-8

#CW
import paddle
import paddle.nn.functional as F
from paddle.vision.transforms import Compose, Normalize

import numpy as np
import cv2

def show_images_diff(original_img,original_label,adversarial_img,adversarial_label):
    import matplotlib.pyplot as plt
    plt.figure()

    plt.subplot(131)
    plt.title('Original')
    plt.imshow(original_img)
    plt.axis('off')

    plt.subplot(132)
    plt.title('Adversarial')
    plt.imshow(adversarial_img)
    plt.axis('off')

    plt.subplot(133)
    plt.title('Adversarial-Original')
    difference = adversarial_img - original_img
    print ("diff shape: ", difference.shape)
    cv2.imwrite("output/cw_diff.png", difference)

    l0=np.where(difference!=0)[0].shape[0]
    l2=np.linalg.norm(difference)
    #print(difference)
    print("l0={} l2={}".format(l0, l2))

    #(-1,1)  -> (0,1)
    difference=difference / abs(difference).max()/2.0+0.5

    plt.imshow(difference,cmap=plt.cm.gray)
    plt.axis('off')
    plt.tight_layout()
    plt.show()

#获取计算设备 默认是CPU
#device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
paddle.set_device('cpu')

#图像加载以及预处理
image_path="./picture/cow.jpeg"
# image_path="./picture/cropped_panda.jpg"
orig = cv2.imread(image_path)[..., ::-1]
print ("orig shape -1: ", orig.shape)#HWC
orig = cv2.resize(orig, (224, 224))
img = orig.copy().astype(np.float32)

mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
img /= 255.0
img = (img - mean) / std
img = img.transpose(2, 0, 1)

img = np.expand_dims(img, axis=0)
print(img.shape)

#使用预测模式 主要影响droupout和BN层的行为
model = paddle.vision.models.vgg16(pretrained=True)
model_s = paddle.Model(model)
print(model_s.summary((-1, 3, 224, 224)))
tmp = model.features(paddle.to_tensor(img, dtype="float32"))
#tmp = model.features[0](paddle.to_tensor(img, dtype="float32"))
#print(tmp.shape)
#tmp = model.features[4](paddle.to_tensor(img, dtype="float32"))
#print(tmp.shape)
##todo avg_pool
tmp = paddle.flatten(tmp, 1)
fc3 = model.classifier(tmp)
#print("Linear-3: ", fc3)
print("Linear-3: ", fc3.shape, type(fc3)) #[1, 1000]

#adam的最大迭代次数 论文中建议10000次 测试阶段1000也可以 1000次可以完成95%的优化工作
#max_iterations = 1000
max_iterations = 100 #mod by zh
#adam学习速率
learning_rate = 0.01
#二分查找最大次数
binary_search_steps = 10
#c的初始值
initial_const = 1e2
confidence = initial_const

#k值
k = 40

#像素值区间
boxmin = -3.0
boxmax = 3.0

#类别数 pytorch的实现里面是1000
num_labels = 1000

#攻击目标标签 必须使用one hot编码
#target_label = 288
target_label = 344
tlab = paddle.eye(num_labels)[target_label]
print("type of tlab: ", type(tlab))

print()

shape = (1,3,224,224)

#c的初始化边界
lower_bound = 0
c = initial_const
upper_bound = 1e10

# the best l2, score, and image attack
o_bestl2 = 1e10
o_bestscore = -1
o_bestattack = [np.zeros(shape)]

# the resulting image, tanh'd to keep bounded from boxmin to boxmax
boxmul = (boxmax - boxmin) / 2.
boxplus = (boxmin + boxmax) / 2.

# output = model(paddle.to_tensor(img, dtype="float32", place=paddle.CUDAPlace(0)))
output = model(paddle.to_tensor(img, dtype="float32"))
orig_label = np.argmax(output)
print("orig_label={}".format(orig_label), type(orig_label))  #345

succ_flag = False
for outer_step in range(binary_search_steps):
    print("o_bestl2={} confidence={}".format(o_bestl2, confidence)  )

    #把原始图像转换成图像数据和扰动的形态
    timg = paddle.to_tensor(np.arctanh((img - boxplus) / boxmul * 0.999999), dtype='float32')
    modifier = paddle.zeros_like(timg, dtype='float32')
    #print (type(modifier))
    # modifier = paddle.to_tensor(modifier, dtype='float32', place=paddle.CUDAPlace(0))
    #print (type(modifier))
    #图像数据的扰动量梯度可以获取
    import pdb
    pdb.set_trace()
    modifier.stop_gradient = False

    #设置为不保存梯度值 自然也无法修改
    for param in model.parameters():
        param.stop_gradient = True

    #定义优化器 仅优化modifier
    #optimizer = torch.optim.Adam([modifier], lr=learning_rate)
    optimizer = paddle.optimizer.Adam(learning_rate=learning_rate, parameters=[modifier])

    for iteration in range(1, max_iterations + 1):
        optimizer.clear_grad()

        #定义新输入
        newimg = paddle.tanh(modifier + timg) * boxmul + boxplus
        #print ("newimg shape: ", newimg.shape)       #[1, 3, 224, 224]
        #print ("new img type: ", type(newimg))
        #newimg = paddle.to_tensor(newimg)
        #print ("new img type: ", type(newimg))
        output = model(newimg)
        pred_label = np.argmax(output)
        print("in iter pred_label={}".format(pred_label))  #345
        #定义cw中的损失函数

        #l2范数
        # print(newimg)
        # print(paddle.tanh(timg) * boxmul + boxplus)
        # loss2 = paddle.dist(newimg, (paddle.tanh(timg) * boxmul + boxplus), p=2)
        loss2 = paddle.sum((newimg - (paddle.tanh(timg) * boxmul + boxplus)) ** 2)

        """
        # compute the probability of the label class versus the maximum other
            real = tf.reduce_sum((tlab)*output,1)
            # 论文中的开源实现
            #other = tf.reduce_max((1-tlab)*output - (tlab*10000),1)
            other = tf.reduce_max((1-tlab)*output)
            loss1 = tf.maximum(0.0, other-real+k)
            loss1 = tf.reduce_sum(const*loss1)
        """
        #paddle.max((1 - tlab) * output) -  paddle.max(output * tlab) +k
        real = paddle.max(output * tlab)
        other = paddle.max((1 - tlab) * output)
        loss1 = other - real + k
        loss1 = paddle.clip(loss1, min=0)

        loss1 = confidence * loss1

        loss = loss1 + loss2

        loss.backward(retain_graph=True)
        pdb.set_trace()
        print(modifier.grad)
        optimizer.step()

        print(modifier)

        l2 = loss2

        sc = output
        print("in loss pred_label: ", np.argmax(sc.numpy()))

        # print out the losses every 10%
        if iteration%(max_iterations//10) == 0:
            print("iteration={} loss={} loss1={} loss2={}".format(iteration,loss,loss1,loss2))

        #if (l2 < o_bestl2) and (np.argmax(sc) == target_label):
        if (l2 < o_bestl2) and (np.argmax(sc.numpy()) != orig_label):
            print("attack success l2={} target_label={} pred_label={}".format(l2, target_label, np.argmax(sc.numpy())))
            o_bestl2 = l2
            o_bestscore = np.argmax(sc.numpy())
            #o_bestattack = newimg.data.cpu().numpy()
            o_bestattack = newimg
            #<class 'paddle.VarBase'> [1, 3, 224, 224]
            print("o_bestattack type: ", type(o_bestattack), o_bestattack.shape)

            ###
            #adv = o_bestattack[0]
            adv = o_bestattack.numpy()
            print("o_bestattack type: ", type(o_bestattack)) #<class 'paddle.VarBase'>
            print("====adv shape: ", adv.shape, type(adv)) # [1, 3, 224, 224]
            adv = np.squeeze(adv)
            print("====after squeeze adv shape: ", adv.shape)#[3, 224, 224]
            adv = adv.transpose(1, 2, 0)
            adv = (adv * std) + mean
            adv = adv * 255.0
            adv = np.clip(adv, 0, 255).astype(np.uint8)
            cv2.imwrite("output/cw_adv_{}_{}.png".format(outer_step, iteration), adv )

            show_images_diff(orig, 0, adv, 0)
            ###
            #succ_flag = True
            #break # mod by zh

    #mod by zh
    #if succ_flag:
    #    break
    confidence_old = -1
    if (o_bestscore == target_label) and o_bestscore != -1:
        #攻击成功 减小c
        upper_bound = min(upper_bound, confidence)
        if upper_bound < 1e9:
                print()
                confidence_old = confidence
                confidence = (lower_bound + upper_bound)/2
    else:
        lower_bound = max(lower_bound, confidence)
        confidence_old = confidence
        if upper_bound < 1e9:
                confidence = (lower_bound + upper_bound) / 2
        else:
                confidence *= 10

    print("outer_step={} confidence {}->{}".format(outer_step, confidence_old, confidence))

print(o_bestattack.shape)
print(img.shape)

adv = o_bestattack.numpy()
#adv=o_bestattack[0]
print(adv.shape)
adv = np.squeeze(adv)
adv = adv.transpose(1, 2, 0)
adv = (adv * std) + mean
adv = adv * 255.0
adv = np.clip(adv, 0, 255).astype(np.uint8)
cv2.imwrite("output/cw_adv.png", adv )

show_images_diff(orig, 0, adv, 0)


#!/usr/bin/env python
from setuptools import setup, find_packages

__doc__ = "Python wrapper for yr weather service"

install_requires = [
    'xmltodict>=0.9.2',
]

version = '0.0.1.dev0'

PACKAGE_NAME = 'py_yr'

setup(
    name=PACKAGE_NAME,
    version=version,
    author="Sindre Nerdal",
    author_email="Sindre.nerdal@gmail.com",
    url="https://github.com/Matmonsen/py_yr",
    keywords="python yr xml",
    description=__doc__,
    #long_description=read('README.rst') + '\n\n' + read('CHANGELOG.rst'),
    packages=find_packages(),
    install_requires=install_requires,
    py_modules=['py_yr'],
    #tests_require=
    #zip_safe=
    #test_suite=
    classifiers=[
        'Development Status :: 2 - Pre-Alpha',
        #'Framework :: ',
        #'Intended Audience :: Developers',
        #'License :: OSI Approved :: BSD License',
        #'Topic :: Internet :: WWW/HTTP',
        #'Programming Language :: Python',
        #'Programming Language :: Python :: 3.3',
        #'Programming Language :: Python :: 3.4',
    ],
)


# Импортируем необходимые библиотеки
import streamlit as st
from PIL import Image
from urllib.request import urlopen
from io import BytesIO

# Импортируем классы из файла exercise_generator
from exercise_generator import TextToDataFrame, DictionaryCreator, ExerciseGenerator

# Задаем ключ API для доступа к словарю
api_key = st.secrets["API_KEY"]

# Определяем функцию для загрузки данных из текста
@st.cache_data
def creating_a_dictionary_and_exercises(text):
    """
    Функция для загрузки данных из текста.

    Аргументы:
    text - текст для обработки.

    Возвращает:
    Кортеж из двух датафреймов:
    - Датафрейм с упражнениями по английскому языку.
    - Датафрейм со словарем.
    """
    # Создаем объект класса TextToDataFrame
    text_to_df = TextToDataFrame(text)

    # Преобразуем текст в датафрейм с упражнениями
    english_exercises_df = text_to_df.txt_to_df()

    # Создаем объект класса DictionaryCreator
    dictionary_creator = DictionaryCreator(api_key)

    # Создаем датафрейм со словарем
    dictionary_df = dictionary_creator.create_dictionary_df(english_exercises_df)

    # Создаем объект класса ExerciseGenerator
    exercise_generator = ExerciseGenerator()

    # Генерируем случайные упражнения
    english_exercises_df = exercise_generator.generate_random_exercises(english_exercises_df)

    # Возвращаем датафреймы с упражнениями и словарем
    return english_exercises_df, dictionary_df

st.title("English exercise generator")

# Создаем виджет для загрузки файлов
uploaded_file = st.file_uploader("Загрузите текстовый файл", type=["txt"])

# Проверяем, был ли загружен файл
if uploaded_file is not None:
    # Читаем содержимое файла и преобразуем его в строку
    text = uploaded_file.read().decode()

    # Загружаем данные из текста
    english_exercises_df, dictionary_df = creating_a_dictionary_and_exercises(text)

    # Отображаем изображение в шапке страницы
    img_url = "https://climbingthedissertationmountain.files.wordpress.com/2018/01/cropped-vladislav-klapin-316711.jpg"
    image_data = urlopen(img_url).read()
    header_image = Image.open(BytesIO(image_data))
    st.image(header_image, use_column_width=True)

    # Отображаем датафрейм со словарем на странице
    st.dataframe(dictionary_df, height=300, width=900)

    # Получаем список типов упражнений из датафрейма с упражнениями
    exercise_types = english_exercises_df["exercise_type"].unique().tolist()

    # Создаем виджет для выбора типов упражнений в боковой панели
    selected_types = st.sidebar.multiselect("Выберите типы упражнений", exercise_types)

    # Фильтруем датафрейм с упражнениями по выбранным типам упражнений
    if not selected_types:
        filtered_df = english_exercises_df.copy()
    else:
        filtered_df = english_exercises_df[english_exercises_df["exercise_type"].isin(selected_types)]

    # Получаем общее количество упражнений в отфильтрованном датафрейме
    total_exercises = len(filtered_df)

    # Получаем текущий индекс и текущий счет из состояния сессии (или задаем значения по умолчанию)
    current_index = st.session_state.get("current_index", 0)
    current_score = st.session_state.get("current_score", 0)

    # Создаем виджет для отображения прогресса выполнения упражнений в боковой панели
    progress_bar = st.sidebar.progress(current_score / total_exercises)

    # Создаем пустой элемент для отображения текущего счета в боковой панели
    score_text = st.sidebar.empty()

    # Отображаем текущий счет в пустом элементе
    score_text.text(f"Решено {current_score} из {total_exercises} упражнений")

    # Отображаем 5 упражнений на странице
    correct_answers = 0
    for i in range(5):

        # Проверяем, есть ли еще упражнения для отображения
        if current_index + i < total_exercises:

            # Отображаем номер упражнения
            st.markdown(f"**Упражнение {current_index + i + 1}**")

            # Получаем тип упражнения из датафрейма
            exercise_type = filtered_df.iloc[current_index + i]["exercise_type"]

            # Отображаем тип упражнения
            st.write(f"Тип упражнения: {exercise_type}")

            # Получаем предложение для упражнения из датафрейма
            exercise_sentence = filtered_df.iloc[current_index + i]["exercise_sentence"]

            # Выделяем пропущенное слово красным цветом
            highlighted_sentence = exercise_sentence.replace("**",
                                                             "",
                                                             1).replace("**", "", 1)

            # Отображаем предложение для упражнения с выделенным пропущенным словом
            st.markdown(f"{highlighted_sentence}", unsafe_allow_html=True)

            # Получаем варианты ответов из датафрейма
            options = filtered_df.iloc[current_index + i]["options"]

            # Добавляем пустой вариант ответа в начало списка
            options.insert(0, "")

            # Создаем выпадающий список с вариантами ответов
            selectbox = st.selectbox(f"Выберите верный ответ", options, key=f"selectbox_{i}")

            # Получаем правильный ответ из датафрейма
            correct_answer = filtered_df.iloc[current_index + i]["correct_answer"]

            # Проверяем, выбрал ли пользователь вариант ответа
            if selectbox:

                # Проверяем, верный ли выбранный вариант ответа
                if selectbox == correct_answer:

                    # Отображаем сообщение об успехе
                    st.success("✅ Верно!")

                    # Увеличиваем счет на 1
                    current_score += 1

                    # Обновляем прогресс выполнения упражнений
                    progress_bar.progress(current_score / total_exercises)

                    # Обновляем отображение текущего счета
                    score_text.text(f"Решено {current_score} из {total_exercises} упражнений")

                    # Увеличиваем счетчик правильных ответов на 1
                    correct_answers += 1

                else:
                    # Отображаем сообщение об ошибке
                    st.error("❌ Неверно!")

                    # Отображаем правильный ответ
                    st.write(f"Правильный ответ: {correct_answer}")

        else:
            break

    # Проверяем, правильно ли решены все 5 упражнений на странице
    if correct_answers == 5:
        # Запускаем анимацию воздушных шаров
        st.balloons()

    # Проверяем, есть ли еще упражнения для отображения на следующей странице
    if current_index + 5 < total_exercises:

        # Создаем кнопку для перехода к следующим 5 упражнениям
        if st.button("Следующие 5 упражнений"):
            # Увеличиваем текущий индекс на 5
            current_index += 5

            # Сохраняем текущий индекс в состоянии сессии
            st.session_state["current_index"] = current_index

            # Сохраняем текущий счет в состоянии сессии
            st.session_state["current_score"] = current_score

            # Перезапускаем приложение
            st.experimental_rerun()

    else:
        # Отображаем сообщение об окончании упражнений
        st.write("Поздравляем! Вы решили все упражнения!")

        # Отображаем итоговый счет
        st.write(f"Ваш итоговый счет: {current_score} из {total_exercises}")

        # Создаем кнопку для начала заново
        if st.button("Начать заново"):
            # Очищаем состояние сессии
            st.session_state.clear()

            # Перезапускаем приложение
            st.experimental_rerun()


from StringIO import StringIO

from fabric import api
from fabric.contrib.files import exists

UPSTART_TEMPLATE = """
description "Pi-Sensor-RPC-Service"
start on runlevel [2345]
stop on runlevel [06]
respawn
respawn limit 10 5

env LOGGLY_TOKEN={loggly_token}
env LOGGLY_SUBDOMAIN={loggly_domain}
env RABBIT_URL={rabbit_url}

script
        cd /home/pi/Pi-Sensor-RPC-Service/app && node main.js
end script
"""

def raspberry_pi():
	api.env.hosts = ['{0}.local'.format(api.prompt('Raspberry Pi:'))]
	api.env.user = 'pi'

def install():
	api.require('hosts', provided_by=[raspberry_pi])

	if exists('/etc/init/sensor-rpc.conf', use_sudo=True):
		print('"sensor-rpc" is already installed, use the "update" task for changes')
		return

	upstart_values = {}
	upstart_values['loggly_token'] = api.prompt("Loggly token:")
	upstart_values['loggly_domain'] = api.prompt("Loggly domain:")
	upstart_values['rabbit_url'] = api.prompt("Rabbit URL:")
	upstart_file = StringIO(UPSTART_TEMPLATE.format(**upstart_values))

	api.sudo('echo Yes, do as I say! | apt-get -y --force-yes install upstart')

	with api.cd('/etc/init'):
		upload = api.put(upstart_file, 'sensor-rpc.conf', use_sudo=True)
		assert upload.succeeded

	api.run('git clone https://github.com/projectweekend/Pi-Sensor-RPC-Service.git')

	with api.cd('~/Pi-Sensor-RPC-Service/app'):
		api.run('npm install')

	api.sudo('service sensor-rpc start')

def update():
	api.require('hosts', provided_by=[raspberry_pi])

	with api.settings(warn_only=True):
		api.sudo('service sensor-rpc stop')

	with api.cd('~/Pi-Sensor-RPC-Service'):
		api.run('git pull origin master')

	with api.cd('~/Pi-Sensor-RPC-Service/app'):
		api.run('npm install')

	api.sudo('service sensor-rpc start')


import re
import os
import config
from pprint import pprint
import json
from sklearn.feature_extraction.text import TfidfVectorizer
from collections import Counter
#from test import EmailData
import email
import matplotlib.pyplot as plt
import numpy as np
import tqdm
TrainProcess = 1
def add_number(rects):
	for i in rects:
		height=round(i.get_height(),2)
		plt.text(i.get_x()+i.get_width()/2,height,height,ha="center",va="bottom")

def EmailData(FilePath):
    fp = open(FilePath, "r", encoding='iso8859', errors='ignore')
    msg = email.message_from_file(fp)
    data = ''
    # 循环信件中的每一个mime的数据块
    for par in msg.walk():
        if not par.is_multipart():  # 这里要判断是否是multipart，是的话，里面的数据是无用的，至于为什么可以了解mime相关知识。
            name = par.get_param("name")  # 如果是附件，这里就会取出附件的文件名
            if name:
                pass
            #   #有附件
            #   # 下面的三行代码只是为了解码象=?gbk?Q?=CF=E0=C6=AC.rar?=这样的文件名
            #   h = email.Header.Header(name)
            #   dh = email.Header.decode_header(h)
            #   fname = dh[0][0]
            #   print ('附件名:', fname)
            #   data = par.get_payload(decode=True) #　解码出附件数据，然后存储到文件中

            #   try:
            #     f = open(fname, 'wb') #注意一定要用wb来打开文件，因为附件一般都是二进制文件
            #   except:
            #     print ('附件名有非法字符，自动换一个')
            #     f = open('aaaa', 'wb')
            #   f.write(data)
            #   f.close()
            else:
                # 不是附件，是文本内容
                data += (' '.join(re.sub('<.*?>|</.*?>|&nbsp|0x\d+|&deg|[a-z|\d]{15,}|\+|-', '',
                                         par.get_payload(decode=False), flags=re.S | re.I).split()))  # 解码出文本内容，直接输出来就可以了。
    fp.close()
    return data

def BayesClassifier():
    if os.path.exists('res.json'):
        os.remove('res.json')
    # HamSet = os.listdir(config.HamPath)
    # SpamSet = os.listdir(config.SpamPath)
    # PHamInAll = len(HamSet)/(len(HamSet)+len(SpamSet))
    # PSpamInall = 1-PHamInAll
    HamSet = []
    SpamSet = []
    with open('indecTrain') as f:
        Line = f.readline()
        while Line:
            TrainData = Line.split()
            if(TrainData[0][0].lower() == 's'):
                SpamSet.append(TrainData[1])
            else:
                HamSet.append(TrainData[1])
            Line = f.readline()
        print("读取完成")
    SpamVocabularyCounter = Counter()
    HamVocabularyCounter = Counter()
    HamVocabularyCounter = FileCounter(HamSet, config.HamPath)
    SpamVocabularyCounter = FileCounter(SpamSet, config.SpamPath)

    # with open('StopWords.txt') as f:
    #     ban_word = f.read().split()
    OnlyInHam = set(HamVocabularyCounter[0].keys(
    ))-set(SpamVocabularyCounter[0].keys())
    OnlyInSpam = set(SpamVocabularyCounter[0].keys(
    ))-set(HamVocabularyCounter[0].keys())
    CorrectHamNum = len(OnlyInSpam)+HamVocabularyCounter[1]
    CorrectSpamNum = len(OnlyInHam)+SpamVocabularyCounter[1]
    for i in OnlyInHam:
        SpamVocabularyCounter[0][i] = 1
    for i in OnlyInSpam:
        HamVocabularyCounter[0][i] = 1
    for i in HamVocabularyCounter[0]:
        HamVocabularyCounter[0][i] /= CorrectHamNum
    for i in SpamVocabularyCounter[0]:
        SpamVocabularyCounter[0][i] /= CorrectSpamNum
    # for i in ban_word:
    #     SpamVocabularyCounter.pop(i,'')
    #     HamVocabularyCounter.pop(i,'')
    print("训练完成")
    with open('res.json', 'w+') as f:
        f.write(json.dumps(
            [SpamVocabularyCounter[0], HamVocabularyCounter[0]]))

def FileCounter(FireDirList, FilePath):
    FileVocabularyCounter = Counter()
    FileVocabularyNum = 0
    # with open('StopWords.txt') as f:
    #     ban_word = f.read().split()
    for i in FireDirList:
        with open(i, 'r') as f:
            data = EmailData(i)
            data = re.sub('[^0-9a-z\-/+ \']', ' ', data, flags=re.I)
            FileVocabularyList = data.lower().strip().split()
            FileVocabularyCounter += Counter(FileVocabularyList)
        global TrainProcess
        print("已完成{}封邮件读取，邮件为{}".format(TrainProcess, i))
        TrainProcess += 1
    for i in ban_word:
        FileVocabularyCounter.pop(i, '')
    for i in FileVocabularyCounter:
        FileVocabularyNum += FileVocabularyCounter[i]
    return (FileVocabularyCounter, FileVocabularyNum)

def BayesClassifier1():
    if os.path.exists('res1.json'):
        os.remove('res1.json')
    HamSet = []
    SpamSet = []
    with open('indecTrain') as f:
        Line = f.readline()
        while Line:
            TrainData = Line.split()
            if(TrainData[0][0].lower() == 's'):
                SpamSet.append(TrainData[1])
            else:
                HamSet.append(TrainData[1])
            Line = f.readline()
        print("读取完成")

    # SpamVocabularyCounter = Counter()
    # HamVocabularyCounter = Counter()

    HamVocabularyCounter = FileCounter1(HamSet, config.HamPath)
    SpamVocabularyCounter = FileCounter1(SpamSet, config.SpamPath)
    ResCounter = Counter()
    # with open('StopWords.txt') as f:
    #     ban_word = f.read().split()
    OnlyInHam = set(HamVocabularyCounter[0].keys()) - \
        set(SpamVocabularyCounter[0].keys())
    OnlyInSpam = set(SpamVocabularyCounter[0].keys()) - \
        set(HamVocabularyCounter[0].keys())
    CorrectEmailNum = len(OnlyInHam)+HamVocabularyCounter[1]+len(OnlyInSpam)
    CorrectHamNum = len(OnlyInSpam)+HamVocabularyCounter[1]
    CorrectSpamNum = len(OnlyInHam)+SpamVocabularyCounter[1]
    PHamInAll = HamVocabularyCounter[1] / \
        (HamVocabularyCounter[1]+SpamVocabularyCounter[1])
    PSpamInall = 1-PHamInAll
    for i in OnlyInHam:
        SpamVocabularyCounter[0][i] = 1
    for i in OnlyInSpam:
        HamVocabularyCounter[0][i] = 1
    for i in SpamVocabularyCounter[0]:
        PWordSpam = SpamVocabularyCounter[0][i]/SpamVocabularyCounter[1]
        PWordHam = HamVocabularyCounter[0][i]/HamVocabularyCounter[1]
        ResCounter[i] = (PWordSpam) / \
            ((PWordHam*PHamInAll+PWordSpam*PSpamInall))
    d_order = sorted(SpamVocabularyCounter[0].items(), key=lambda x: x[1], reverse=True)[:-1]
    # pprint(d_order)
    SpamResDic = {}
    HamResDic={}
    for i in d_order:
        SpamResDic[i[0]] = ResCounter[i[0]]
    SpamResDic['spam_prob']=PSpamInall

    # for i in ban_word:
    #     SpamVocabularyCounter.pop(i,'')
    #     HamVocabularyCounter.pop(i,'')
    with open('res1.json', 'w+') as f:
        f.write(json.dumps(SpamResDic))

def FileCounter1(FireDirList, FilePath):
    FileVocabularyCounter = Counter()
    FileVocabularyNum = 0
    ResCounter = Counter()
    # with open('StopWords.txt') as f:
    #     ban_word = f.read().split()
    for i in tqdm.tqdm(FireDirList):
        with open(i, 'r') as f:
            data = EmailData(i)
            data = re.sub('[^0-9a-z\-/+ \']', ' ', data.lower(), flags=re.I)
            FileVocabularyList = list(set(data.lower().strip().split()))
            FileVocabularyCounter += Counter(FileVocabularyList)
        global TrainProcess
        #print("已完成{}封邮件读取，邮件为{}".format(TrainProcess, i))
        TrainProcess += 1
    FileVocabularyNum += len(FileVocabularyList)
    for i in ban_word:
        FileVocabularyCounter.pop(i, '')
    for i in FileVocabularyCounter:
        if FileVocabularyCounter[i] != 1 and len(i)>=3:
            FileVocabularyNum += FileVocabularyCounter[i]
            ResCounter[i] = FileVocabularyCounter[i]
    return (ResCounter, FileVocabularyNum)

def FileVocabularDfitf(FireDirList, FilePath):
    FileVocabularyCounter = Counter()
    FileVocabularyNum = 0
    data = ''
    for i in FireDirList:
        with open(FilePath+'/'+i, 'r') as f:
            data += f.read()
    # with open('StopWords.txt') as f:
    #     ban_word = f.read().split()
    copus = [data]

    vector = TfidfVectorizer(stop_words=ban_word)
    tfidf = vector.fit_transform(copus)
    res_dic = {}
    wordlist = vector.get_feature_names()
    weightlist = tfidf.toarray()
    for i in range(len(weightlist)):
        print("-------在中第", i, "段评论的词语 tf-idf 权重------")
        for j in range(len(wordlist)):
            res_dic[wordlist[j]] = weightlist[i][j]
        d_order = sorted(res_dic.items(), key=lambda x: x[1], reverse=True)
        pprint(d_order[0:15])
        res_dic = {}
    res_word = []
    for i in range(15):
        res_word.append(d_order[i][0])
    return res_word

def GetRes(FilePath,ham,spam):
    tmp = EmailData(FilePath)
    tmp = re.sub('[^0-9a-z\-/+ \']', ' ', tmp, flags=re.I).lower().split()
    # for i in ban_word:
    #     tmp.pop(i, '')
    count = 1
    for i in tmp:
        if spam.__contains__(i):
            var1 = spam[i]/ham[i]
            count *= var1
    if count > 1.356:
        return "Spam"
    else:
        return "ham"

# def GetRes1(FilePath):
#     tmp = EmailData(FilePath)

#     # f=open(FilePath,'r',errors='ignore')
#     # tmp=f.read()
#     # f.close()
#     tmp = re.sub('[^0-9a-z\-/+ \']', ' ', tmp, flags=re.I).lower().split()
#     count = 1
#     for i in tmp:
#         if ham.__contains__(i):
#             var1 = spam[i]/ham[i]
#             count *= var1
#     if count > 1000:
#         return "Spam"
#     else:
#         return "ham"
def GetRes1(FilePath,res):
    #print(FilePath)
    tmp = EmailData(FilePath)
    count=1
    flag=0
    tmp = ((re.sub('[^0-9a-z\-/+ \']', ' ', tmp, flags=re.I).lower().split()))
    # for i in ban_word:
    #     tmp.pop(i, '')
    # d_order = sorted(tmp.items(), key=lambda x: x[1], reverse=True)[:30]

    for i in tmp:
        # if len(i[0])>30:
        #     return ("Spam",9999)

        if spam.__contains__(i):
            count *= spam[i]
            # if spam[i]==1:
            #     count1=0.01
            # else:
            #     count1=1-spam[i]
            # res*=(spam[i]/count1)
            flag += 1
    # res=count/count1
    # if len(tmp)>600:
    #     return ("ham",9999)
    count*=data['spam_prob']
    if count>1.3:
        return ("Spam",count)
    else:
        return ("ham",count)

def TestRes():
    count = 0
    TmpValue = 0
    count_flase=0
    count_flase1=0
    count_right=0
    count_right1=0
    TestNum=0
    f=open('res.json')
    data = json.loads(f.read())
    spam = data[0]
    ham = data[1]
    with open('indecTest') as f:
        Line = f.readline()
        while Line and TmpValue<10000:
            TestData = Line.split()
            GuessRes = GetRes(TestData[1],ham,spam)
            TmpValue += 1
            if GuessRes.lower() == TestData[0].lower():
                count += 1
                if(TestData[0].lower()=='spam'):
                    count_right+=1  #tp
                else:
                    count_right1+=1
               # print("预测结果为{}正确".format(TestData[0]))
            #     if(count % 1000 == 0):
            #         print("预测准确率为{}".format(str(count/TmpValue)))
            # else:
            #     pass
               # print("预测结果为{},但真实结果为{}".format(GuessRes,TestData[0]))
            else:
                if(TestData[0].lower()=='spam' ):
                    count_flase+=1      #fn
                else:
                    count_flase1+=1
            Line = f.readline()
        #print("预测准确率为{}".format(str(count/TmpValue)))
        accuracy=count/TmpValue
        recall=count_right/(count_right+count_flase)
        precision=count_right/(count_right+count_flase1)
        Fmeansure=2*precision*recall/(precision+recall)
        print("accuracy: {}".format(str(accuracy)))
        print("recall: {}".format(str(recall)))
        print("precision: {}".format(str(precision)))
        print("f-meansure: {}".format(str(Fmeansure)))
        ResList=[accuracy,recall,precision,Fmeansure]
        NameList=['accuracy','recall','precision','f-meansure']
        a=plt.bar([0,1,2,3],ResList,tick_label=NameList)
        add_number(a)
        # plt.show()

def TestRes1():
    x=[]
    y=[]
    count = 0
    TmpValue = 0
    count_flase=0
    count_flase1=0
    count_right=0
    count_right1=0
    with open('indecTest') as f:
        Line = f.readline()
        while Line and TmpValue<10000:
            TestData = Line.split()
            GuessRes = GetRes1(TestData[1],TestData[0])
            TmpValue += 1
            if GuessRes[0].lower() == TestData[0].lower():
                count += 1
                if(TestData[0].lower()=='spam'):
                    count_right+=1  #tp
                else:
                    count_right1+=1

               # print("预测结果为{}正确".format(TestData[0]))
            #     if(count % 1000 == 0):
            #         print("预测准确率为{}".format(str(count/TmpValue)))
            #         # return (count/TmpValue)
            #         pass
            # else:
            #     pass
                #print(TestData[1])
            else:
                if(TestData[0].lower()=='spam' ):
                    count_flase+=1      #fn
                else:
                    count_flase1+=1     #fp
                #print(GuessRes[1])
                #print("预测结果为{},但真实结果为{}".format(GuessRes,TestData[0]))
                # break
            if count_right>0:
                x.append(count_right/(count_right+count_flase))
                y.append(count_right/(count_right+count_flase1))
            Line = f.readline()
        plt.plot(x,y)
        plt.show()
        accuracy=count/TmpValue
        recall=count_right/(count_right+count_flase)
        precision=count_right/(count_right+count_flase1)
        Fmeansure=2*precision*recall/(precision+recall)
        print("accuracy: {}".format(str(accuracy)))
        print("recall: {}".format(str(recall)))
        print("precision: {}".format(str(precision)))
        print("f-meansure: {}".format(str(Fmeansure)))
        ResList=[accuracy,recall,precision,Fmeansure]
        NameList=['accuracy','recall','precision','f-meansure']
        a=plt.bar([0,1,2,3],ResList,tick_label=NameList)
        add_number(a)
        # plt.show()

    return count/TmpValue

if __name__ == '__main__':
    #    wordlist=FileVocabularDfitf(os.listdir(config.SpamPath),config.SpamPath)
    with open('StopWords.txt') as f:
        ban_word = f.read().split()
    #BayesClassifier1()
    #     data = json.loads(f.read())
    #     spam = data[0]
    #     ham = data[1]
    # TestRes()
    x = np.arange(0.98, 1, 0.001)
    # print(x)
    y = []
    with open('res1.json') as f:
        data = json.loads(f.read())
    # for j in tqdm.tqdm(x):
    #     spam={}
    #     for i in data:
    #         if data[i]>=j:
    #             spam[i]=data[i]
    #     y.append(TestRes1())
    # plt.plot(x,y)
    # plt.show()
    # pprint(y)
    # print(np.max(y))
    spam = {}
    d_order = sorted(data.items(), key=lambda x: x[1], reverse=True)[:3000]
    #pprint(d_order)
    for i in d_order:
        spam[i[0]]=i[1]
    spam['spam_prob']=data['spam_prob']
    # #spam=data
    # print(d_order[-1])
    import time
    time1=time.time()
    TestRes1()
    time2=time.time()
    print("*"*100)
    TestRes()
    time3=time.time()
    print(time2-time1)
    print(time3-time2)


import torch
import torchvision
from torchvision import transforms,datasets,models
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

train_set = datasets.MNIST('~/torch_datasets', train=True, download=True)
test_set = datasets.MNIST('~/torch_datasets', train=False, download=True)
train_img = train_set.data.numpy()
test_img = test_set.data.numpy()

def train_loader_fn():
    train_images = torch.Tensor(train_img).view(train_img.shape[0],1,28,28).to(device)
    return train_images
def test_loader_fn():
    test_images = torch.Tensor(test_img).view(test_img.shape[0],1,28,28).to(device)
    return test_images


import importlib
import inspect
import os

__all__ = [
    'abs_path',
    'asset_path',
    'fully_qualified_name',
    'is_asset_path',
    'is_module_path',
    'is_object_path',
]

def abs_path(path):
    """Get abs. path for ``path``.

    ``path`` may be a relative or absolute file system path or an asset
    path. If ``path`` is already an abs. path, it will be returned as
    is. Otherwise, it will be converted into a normalized abs. path.

    """
    if not os.path.isabs(path):
        if is_asset_path(path):
            path = asset_path(path)
        else:
            path = os.path.expanduser(path)
            path = os.path.normpath(os.path.abspath(path))
    return path

def asset_path(path, *rel_path):
    """Get absolute path to asset in package.

    ``path`` can be just a package name like 'tangled.web' or it can be
    a package name and a relative file system path like
    'tangled.web:some/path'.

    If ``rel_path`` is passed, it will be appended to the base rel. path
    in ``path``.

    Examples::

        >>> asset_path('tangled.util')
        '.../tangled/tangled'
        >>> asset_path('tangled.util:')
        '.../tangled/tangled'
        >>> asset_path('tangled.util:x')
        '.../tangled/tangled/x'
        >>> asset_path('tangled.util', 'x')
        '.../tangled/tangled/x'
        >>> asset_path('tangled.util:x', 'y')
        '.../tangled/tangled/x/y'

    """
    if not (is_asset_path(path) or is_module_path(path)):
        raise ValueError('Path is not an asset path: %s' % path)
    if ':' in path:
        package_name, base_rel_path = path.split(':')
        rel_path = (base_rel_path,) + rel_path
    else:
        package_name = path
    package = importlib.import_module(package_name)
    if not hasattr(package, '__file__'):
        raise ValueError("Can't compute path relative to namespace package")
    package_path = os.path.dirname(package.__file__)
    path = os.path.join(package_path, *rel_path)
    path = os.path.normpath(os.path.abspath(path))
    return path

def fully_qualified_name(obj):
    """Get the fully qualified name for an object.

    Returns the object's module name joined with its qualified name. If
    the object is a module, its name is returned.

    >>> fully_qualified_name(object)
    'builtins.object'
    >>> import tangled.util
    >>> fully_qualified_name(tangled.util)
    'tangled.util'

    """
    if inspect.ismodule(obj):
        return obj.__name__
    return '{}.{}'.format(obj.__module__, obj.__qualname__)

def is_asset_path(path) -> bool:
    """Is ``path`` an asset path like ``package.module:path``?

    If ``path`` is absolute, it will not be considered an asset path.
    Otherwise, it will be considered an asset path if it contains a
    colon *and* the module path contains only valid Python identifiers.
    The file system path to the right of the colon can be empty or any
    string (it's ignored here).

    Examples::

        >>> is_asset_path('/some/abs/path')
        False
        >>> is_asset_path('rel/path')
        False
        >>> is_asset_path('package')
        False
        >>> is_asset_path('package:')
        True
        >>> is_asset_path('package.subpackage:rel/path')
        True
        >>> is_asset_path('package.subpackage:')
        True
        >>> is_asset_path('package.subpackage:rel/path')
        True
        >>> is_asset_path('base.ini')
        False

    """
    if os.path.isabs(path) or ':' not in path:
        return False
    module_path, _ = path.split(':', 1)
    return is_module_path(module_path)

def is_module_path(path) -> bool:
    """Is ``path`` a module path like ``package.module``?

    Examples::

        >>> is_module_path('package')
        True
        >>> is_module_path('package.module')
        True
        >>> is_module_path('.module')
        True
        >>> is_module_path('package.module:obj')
        False
        >>> is_module_path('a/b')
        False
        >>> is_module_path('/a/b')
        False

    """
    return all(p.isidentifier() for p in path.lstrip('.').split('.'))

def is_object_path(path) -> bool:
    """Is ``path`` an object path like ``package.module:obj.path``?

    Examples::

        >>> is_object_path('package.module:obj')
        True
        >>> is_object_path('.module:obj')
        True
        >>> is_object_path('package')
        False
        >>> is_object_path('package:')
        False
        >>> is_object_path('a/b')
        False
        >>> is_object_path('/a/b')
        False

    """
    if ':' not in path:
        return False
    pkg_path, *object_path = path.split(':', 1)
    if object_path:
        object_path = object_path[0]
        return is_module_path(pkg_path) and is_module_path(object_path)
    return False


# @Time    : 2020-6-28
# @Author  : Qiushi Li
# @File    : 8_test_decom.py

import os
import sys
import argparse
os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID'
os.environ['CUDA_VISIBLE_DEVICES'] = '6'

from setup_task_verify_experiment import *

parser = argparse.ArgumentParser("Test_decom_Params")
parser.add_argument('--log_root', type=str, default='../../EXP', help='log dir')
parser.add_argument('--work_name', type=str, default='test_decom', help='work name, 6_finetune or 7_train_from_scratch')
parser.add_argument('--data_path', type=str, default='', help='dataset path')
parser.add_argument('--load_path', type=str, default='', help='model to be loaded')
parser.add_argument('--load_config', type=str, default='', help='config file path')
args = parser.parse_args()

assert args.load_path != '', "Please specify args.load_path !"
assert args.load_config != '', "Please specify args.load_config !"

if __name__ == '__main__':
    TEST_TXT = BB_TEST_TXT
    COVER_ROOT = os.path.join(args.data_path, 'bb_256_qf75/')
    STEGO_ROOT = os.path.join(args.data_path, 'bb_256_qf75_juniward04/')
    test_gen = snc64.partial(snc64.gen_valid, \
                        COVER_ROOT, STEGO_ROOT, \
                            TEST_TXT, TEST_TXT)

    LOG_DIR = os.path.join(args.log_root, time.strftime("%Y%m%d-%H%M%S-") + args.work_name) # path for a log direcotry

    LOAD_CKPT = args.load_path

    decom_acc_log_path = os.path.join(LOG_DIR, 'logs')
    if not os.path.exists(decom_acc_log_path):
        os.makedirs(decom_acc_log_path)
    decom_acc_log_path = os.path.join(decom_acc_log_path, 'test_decom_acc.txt')

    with open(TEST_TXT) as f:
        test_cover = f.readlines()
        test_cover_list = [a.strip() for a in test_cover]
    test_ds_size = len(test_cover_list) * 2
    print ('test_ds_size: %i'%test_ds_size)

    if test_ds_size % test_batch_size != 0:
        raise ValueError("change batch size for testing!")

    snd.test_dataset(snd.SRNetDecom, test_gen, test_batch_size, test_ds_size, LOAD_CKPT, decom_acc_log_path, args.load_config)


import discord
import os
from discord.ext import commands
from profanity_check import predict
import joblib
import sklearn.externals

class DiscordBot(commands.Bot):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

bot = DiscordBot(command_prefix='>')

@bot.event
async def on_message(message):
    # check for profanity
    if predict([message.content])[0]:
        # delete the message
        await message.delete()
        # send a warning message to the user
        await message.author.send("Please watch your language.")

@bot.event
async def on_ready():
    print(f'{bot.user.name} has connected to Discord!')

@bot.command(name='kick', help='Kicks the mentioned user')
async def kick(ctx, member: discord.Member):
    await ctx.guild.kick(member)
    await ctx.send(f'{member} has been kicked')

@bot.command()
async def slowmode(ctx, time: int):
    channel = ctx.channel
    await channel.edit(slowmode_delay=time)
    await ctx.send(f'Slowmode for this channel has been set to {time} seconds.')

@bot.command()
async def warn(ctx, member: discord.Member, *, reason: str):
    await ctx.send(f'{member.mention} has been warned for {reason}')

@bot.command(name='Ban', help='Bans the mentioned user')
async def ban(ctx, member: discord.Member):
    await ctx.guild.ban(member)
    await ctx.send(f'{member} has been banned')

@bot.command(name='Mute', help='Mutes the mentioned user')
async def mute(ctx, member: discord.Member):
    mute_role = discord.utils.get(ctx.guild.roles, name='Muted')
    await member.add_roles(mute_role)
    await ctx.send(f'{member} has been muted')

@bot.command(name='Purge', help='Deletes a specified number of messages')
async def purge(ctx, amount: int):
    await ctx.channel.purge(limit=amount)
    await ctx.send(f'{amount} messages have been deleted')

bot.run(os.environ['BOT_TOKEN'])


# Author: kk.Fang(fkfkbill@gmail.com)

__all__ = [
    "OracleOnlineSQLTextIssueSingleAnalyse"
]

from typing import Generator, List

import rule.const
from ....issue.base import *
from .sql_text import *
from ....single_sql import SingleSQLForOnline
from ....cmdb import OracleCMDB
from ....plain_db import *
from rule.adapters import RuleAdapterSQL
from ....capture import OracleSQLText

@OracleOnlineIssue.need_collect()
class OracleOnlineSQLTextIssueSingleAnalyse(OracleOnlineSQLTextIssue):
    """sql文本问题(逐个分析)"""

    ENTRIES = (rule.const.RULE_ENTRY_ONLINE_SINGLE,)

    @classmethod
    def _single_rule(cls,
                     the_rule,
                     sqls: List[SingleSQLForOnline],
                     cmdb_connector: OraclePlainConnector,
                     entries: List[str],
                     the_cmdb: OracleCMDB,
                     task_record_id: int,
                     schema_name: str,
                     **kwargs) -> Generator["OracleOnlineSQLTextIssue", None, None]:
        for single_sql in sqls:
            ret = RuleAdapterSQL(the_rule).run(
                entries=entries,

                cmdb=the_cmdb,
                single_sql=single_sql
            )
            docs = cls.pack_rule_ret_to_doc(the_rule, ret)
            cls.post_analysed(
                docs=docs,
                task_record_id=task_record_id,
                schema_name=schema_name
            )
            yield from docs


#!/usr/bin/env python

#Licensed under the MIT licence
#To end the Programme use "C+Ctrl" at the same time.

#pretty obvious, but you'll want these dependencies to be installed for this script to work

# This imports the Raspberry Pi's GPIO library but as GPIO so we don't need to be repeating it.
import RPi.GPIO as GPIO

#This is used to sleep the code a the end.
import time

#Requests is the ONE that is need to be installed with "sudo apt-get install python-requests" this lets us to grab and interpret PHP
import requests

import os

def cls():
    os.system(['clear','cls'][os.name == 'nt'])

#Uncomment this is you don't want to get the GPIO warning. This is useful if you changing things around constantly and are ending the code.
#GPIO.setwarnings(False)

DEBUG = 1

#this defines the pins for each ONE
ONE = 23
TWO = 24
THREE = 25

#sets the mode for the pin out
GPIO.setmode(GPIO.BCM)

#defines if the pins are inputs or out puts
GPIO.setup(ONE, GPIO.OUT)
GPIO.setup(TWO, GPIO.OUT)
GPIO.setup(THREE, GPIO.OUT)

#replace the http://YourWebsite.com/test.php with where your test.php is located.
response = requests.get('http://yourwebsite.com/text.php')

#replace the http://YourWebsite.com/test.php with where your test.php is located.
while response.text != 'exit':
    response = requests.get('http://yourwebsite.com/text.php')

#This says ONE is the first digit TWO is the second and THREE is the last
    colrs = response.text
    one = colrs[0]
    two = colrs[1]
    three = colrs[2]

    cls()

    if response.text == 'exit':
        print (colrs)
    else:
        print ("one" + one + ", "),
        print ("two" + two + ", "),
        print ("three" + three)
        print

# This says if the number is 1 then the the light is on else if it is 0 the light is off.
    if one == '1':
        GPIO.output(ONE,True)
    else:
        GPIO.output(ONE,False)

    if two  == '1':
        GPIO.output(TWO,True)
    else:
        GPIO.output(TWO,False)

    if three  == '1':
        GPIO.output(THREE,True)
    else:
        GPIO.output(THREE,False)

#This can be removed but this time is to stop the web server from being overloaded with requests.
    time.sleep(0.15)


"""Views for Canopact carbon dashboard.

"""

import datetime
import json

from canopact.blueprints.carbon.models.carbon import Carbon
from canopact.blueprints.carbon.models.expense import Expense
from canopact.blueprints.carbon.models.report import Report
from canopact.blueprints.carbon.models.route import Route
from canopact.blueprints.carbon.forms import (
    SearchForm,
    RouteForm,
    JourneysForm,
    DateForm
)
from canopact.blueprints.billing.decorators import (
    subscription_required
)
from canopact.blueprints.user.decorators import (
    email_confirm_required,
    expensify_required
)
from canopact.extensions import db, cache
from flask import (
    Blueprint,
    current_app,
    flash,
    redirect,
    render_template,
    url_for,
    request,
)
from flask_login import current_user, login_required
from sqlalchemy import text

carbon = Blueprint('carbon', __name__, template_folder='templates')

# Dashboard -------------------------------------------------------------------
@cache.cached(timeout=300, key_prefix="date_input")
def get_date_input():
    """Get value from date input form and convert to date.

    Caches date input so that it persists across different levels of
        aggregation.

    Returns:
        date (datetime.datetime): date from input form.

    """
    date = request.form.get('date')
    date = datetime.datetime.strptime(date, "%Y-%m-%d").date()

    return date

@carbon.route('/carbon/dashboard/<agg>', methods=['GET', 'POST'])
# @expensify_required()
@subscription_required
@email_confirm_required()
@login_required
def dashboard(agg):
    """Renders template for the carbon dashboard

    Args:
        agg (str): level of aggregation for the dashboards

    """
    form = DateForm()

    # Date input.
    if form.validate_on_submit():
        cache.clear()
        date = get_date_input()
    else:
        try:
            date = get_date_input()
        except BaseException:
            date = datetime.date.today()

    # Update data in form.
    form.date.data = date

    # KPI cards.
    emissions, prev_emissions, emissions_change = \
        Carbon.emissions_metrics(current_user, agg=agg, date=date)
    per_journeys, prev_per_journeys, per_journeys_change = \
        Carbon.per_journeys_metrics(current_user, agg=agg, date=date)
    cost, prev_cost, cost_change = \
        Carbon.cost_metrics(current_user, agg=agg, date=date)
    cost_per_journey, prev_cost_per_journey, cost_per_journey_change = \
        Carbon.cost_per_journey_metrics(current_user, agg=agg, date=date)

    # Charts.
    journeys = Carbon.group_and_count_journeys_monthly(current_user, agg=agg,
                                                       date=date)
    monthly_carbon = Carbon.group_and_sum_emissions_monthly(current_user,
                                                            agg=agg,
                                                            prev_months=8,
                                                            date=date)
    line = Carbon.emissions_metrics_monthly(current_user, agg=agg,
                                            prev_months=8,
                                            emissions=monthly_carbon,
                                            date=date)

    # Line chart filters.
    emissions_line = line["datasets"]["emissions"]
    per_journey_line = line["datasets"]["per_journey"]
    per_km_line = line["datasets"]["per_km"]

    # Tables.
    transports = Carbon.group_and_count_transport(current_user, agg=agg,
                                                  as_list=True, date=date)
    routes = Carbon.group_and_count_routes(current_user, agg=agg, as_list=True,
                                           date=date)

    return render_template('dashboard/index.html', emissions=emissions,
                           emissions_change=emissions_change,
                           emissions_per_journeys=per_journeys,
                           per_journeys_change=per_journeys_change,
                           cost=cost,
                           cost_change=cost_change,
                           cost_per_journey=cost_per_journey,
                           cost_per_journey_change=cost_per_journey_change,
                           journeys=json.dumps(journeys),
                           monthly_carbon=json.dumps(monthly_carbon),
                           emissions_line=json.dumps(emissions_line),
                           per_journey_line=json.dumps(per_journey_line),
                           per_km_line=json.dumps(per_km_line),
                           routes=routes, transports=transports,
                           form=form)

# Routes Cleaner --------------------------------------------------------------
def get_routes():
    """Fetches routes that do not have a valid origin/destination.

    Orders routes according to sorting selected on the router cleaner table.

    Returns:
        routes (sqlachemy.Query): routes which require origin/destination
            values to be updated.

    """

    sort_by = Route.sort_by(request.args.get('sort', 'expense_created_date'),
                            request.args.get('direction', 'desc'))
    order_values = 'routes.{0} {1}'.format(sort_by[0], sort_by[1])

    routes = db.session.query(Route.id,
                              Route.created_on,
                              Route.expense_id,
                              Route.expense_category,
                              Report.report_name,
                              Expense.expense_merchant,
                              Expense.expense_comment,
                              Expense.expense_created_date) \
        .join(Expense, Route.expense_id == Expense.expense_id) \
        .join(Report, Expense.report_id == Report.report_id) \
        .filter(Expense.user_id == current_user.id) \
        .filter(Route.route_category != 'unit') \
        .filter((Route.origin.is_(None) | Route.destination.is_(None)) |
                (Route.invalid == 1)) \
        .filter(Route.search(request.args.get('q', ''))) \
        .order_by(text(order_values)) \
        .distinct()

    return routes

@carbon.route('/carbon/cleaner')
# @expensify_required()
@subscription_required
@email_confirm_required()
@login_required
def cleaner():
    """Retrieves all expenses with an invalid and renders cleaner template."""

    search_form = SearchForm()
    routes = get_routes()

    return render_template('cleaner/cleaner.html', form=search_form,
                           routes=routes)

@carbon.route('/carbon/cleaner/edit', methods=['GET', 'POST'])
# @expensify_required()
@email_confirm_required()
@login_required
def routes_edit():
    """Opens editor mode to let user enter in valid origin and destination.

    """
    routes = get_routes()
    total = int(routes.count())  # total number of invalid routes.

    journeys_form = JourneysForm()

    # Key for Google Autocomplete API.
    key = current_app.config['DISTANCE_KEY']

    # Iterate over each route submitted on the cleaner.
    if journeys_form.validate_on_submit():
        for i, entry in enumerate(journeys_form.journeys.entries):
            # Get the route id in order to instantiate a Route object.
            id = journeys_form.ids[i]
            r = Route.query.get(id)

            # Parse the fields from the FieldList.
            if entry.data['origin'] == '':
                origin = None
            else:
                origin = entry.data['origin']

            if entry.data['destination'] == '':
                destination = None
            else:
                destination = entry.data['destination']

            if entry.data['return_type']:
                return_type = 'return'
            else:
                return_type = None

            # Update and save the Route model.
            if all(v is not None for v in [origin, destination]):
                r.origin = origin
                r.destination = destination
                r.return_type = return_type
                r.invalid = 0  # Change the invalid flag.
                r.update_and_save(Route, id=id)

        # Clear journeys form.
        while len(journeys_form.journeys.entries) > 0:
            journeys_form.journeys.pop_entry()
        while len(journeys_form.ids) > 0:
            journeys_form.ids.pop()

        flash('Routes has been saved successfully.', 'success')
        return redirect(url_for('carbon.cleaner'))
    else:
        # Clear journeys form.
        while len(journeys_form.journeys.entries) > 0:
            journeys_form.journeys.pop_entry()
        while len(journeys_form.ids) > 0:
            journeys_form.ids.pop()

        for route in routes:
            route_form = RouteForm()
            route_form.origin = None
            route_form.destination = None
            route_form.return_type = None
            journeys_form.ids.append(route.id)
            journeys_form.journeys.append_entry(route_form)

    return render_template('cleaner/edit.html', form=journeys_form,
                           routes=routes, total=total, key=key)


import asyncio

from aiogram import types, Dispatcher
from create_bot import dp, bot
from sheets_settings import read_sheets, new_data_price, update, add_new_data, new_data_sku
from aiogram.dispatcher.filters import Text
from key.buttons import menu_info
from config import AUTHORIZED_USER_ID

async def base_commands(message: types.Message):
    if str(message.from_user.id) in AUTHORIZED_USER_ID:
        await bot.send_message(message.from_user.id, 'Welcome', reply_markup=menu_info)
    else:
        await bot.send_message(chat_id=message.from_user.id, text="Sorry, you are not authorized to use this bot.")

async def update_google_sheets(message: types.Message):
    if str(message.from_user.id) in AUTHORIZED_USER_ID:
        await bot.send_message(message.from_user.id, 'Немного подождите')
        name_excel, price_excel = await read_sheets()
        await add_new_data(name_excel, price_excel)
        name_excel, price_excel = await read_sheets()
        new_sku = await new_data_sku(name_excel, price_excel)
        await update(new_sku)
        await bot.send_message(message.from_user.id, 'Данные были обновлены')
    else:
        await bot.send_message(chat_id=message.from_user.id, text="Sorry, you are not authorized to use this bot.")

async def new_data(message: types.Message):
    await bot.send_document(chat_id=message.from_user.id, document=open('new_sku_data.xlsx', 'rb'))

def register_admin(dp : Dispatcher):
    dp.register_message_handler(base_commands, commands=['start'])
    dp.register_message_handler(update_google_sheets, Text(equals='обновить данные', ignore_case=True))
    dp.register_message_handler(new_data, Text(equals='новые данные', ignore_case=True))


"""
Using Cartopy and AxesGrid toolkit
----------------------------------

This example demonstrates how to use cartopy `~cartopy.mpl.geoaxes.GeoAxes`
with `~mpl_toolkits.axes_grid1.axes_grid.AxesGrid` from the
:mod:`mpl_toolkits.axes_grid1`. The script constructs an ``axes_class`` kwarg
with Plate Carree projection and passes it to the
`~mpl_toolkits.axes_grid1.axes_grid.AxesGrid` instance. The
`~mpl_toolkits.axes_grid1.axes_grid.AxesGrid` built-in tick labelling
is not used, and instead a standard procedure of creating grid lines is used.
Then some fake data is plotted.
"""
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import AxesGrid
import numpy as np

import cartopy.crs as ccrs
from cartopy.mpl.geoaxes import GeoAxes
from cartopy.mpl.ticker import LatitudeFormatter, LongitudeFormatter

def sample_data_3d(shape):
    """Return `lons`, `lats`, `times` and fake `data`"""
    ntimes, nlats, nlons = shape
    lats = np.linspace(-np.pi / 2, np.pi / 2, nlats)
    lons = np.linspace(0, 2 * np.pi, nlons)
    lons, lats = np.meshgrid(lons, lats)
    wave = 0.75 * (np.sin(2 * lats) ** 8) * np.cos(4 * lons)
    mean = 0.5 * np.cos(2 * lats) * ((np.sin(2 * lats)) ** 2 + 2)

    lats = np.rad2deg(lats)
    lons = np.rad2deg(lons)
    data = wave + mean

    times = np.linspace(-1, 1, ntimes)
    new_shape = data.shape + (ntimes, )
    data = np.rollaxis(data.repeat(ntimes).reshape(new_shape), -1)
    data *= times[:, np.newaxis, np.newaxis]

    return lons, lats, times, data

def main():
    projection = ccrs.PlateCarree()
    axes_class = (GeoAxes,
                  dict(projection=projection))

    lons, lats, times, data = sample_data_3d((6, 73, 145))

    fig = plt.figure()
    axgr = AxesGrid(fig, 111, axes_class=axes_class,
                    nrows_ncols=(3, 2),
                    axes_pad=0.6,
                    cbar_location='right',
                    cbar_mode='single',
                    cbar_pad=0.2,
                    cbar_size='3%',
                    label_mode='keep')

    for i, ax in enumerate(axgr):
        ax.coastlines()
        ax.set_xticks(np.linspace(-180, 180, 5), crs=projection)
        ax.set_yticks(np.linspace(-90, 90, 5), crs=projection)
        lon_formatter = LongitudeFormatter(zero_direction_label=True)
        lat_formatter = LatitudeFormatter()
        ax.xaxis.set_major_formatter(lon_formatter)
        ax.yaxis.set_major_formatter(lat_formatter)

        p = ax.contourf(lons, lats, data[i, ...],
                        transform=projection,
                        cmap='RdBu')

    axgr.cbar_axes[0].colorbar(p)

    plt.show()

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-

import wx
import os.path

TITLE_TXT = """Huayra Update"""
ABOUT_TXT = """Te avisa cuando hay actualizaciones del sistema disponibles."""
VERSION_TXT = """Versión: 0.4"""

class AboutDialog(wx.Frame):
    def __init__(self, parent=None):
        super(AboutDialog, self).__init__(
            parent=parent,
            title=u'Acerca de',
        )

        img = os.path.join(wx.GetApp().app_path, 'media', 'fondo.png')
        bmp = wx.Image(img, wx.BITMAP_TYPE_ANY).ConvertToBitmap()

        panel = wx.Panel(self)
        wx.StaticBitmap(panel, -1, bmp, (0, 0))

        font = wx.Font(12, wx.MODERN, wx.NORMAL, wx.NORMAL, False, u'Asap')

        txt = wx.StaticText(panel, id=-1, label=TITLE_TXT, pos=(115, 40))
        txt.SetFont(
            wx.Font(12, wx.MODERN, wx.NORMAL, wx.BOLD, False, u'Asap')
        )

        txt1 = wx.StaticText(panel, id=-1, label=ABOUT_TXT, pos=(115, 58))
        txt1.Wrap(200)
        txt1.SetFont(font)

        txt2 = wx.StaticText(panel, id=-1, label=VERSION_TXT, pos=(115, 120))
        txt2.Wrap(200)
        txt2.SetFont(font)

        txt.SetForegroundColour((0,0,0))
        txt1.SetForegroundColour((0,0,0))
        txt2.SetForegroundColour((0,0,0))

        self.SetMinSize((350, 250))
        self.SetMaxSize((350, 250))

        w, h = wx.DisplaySize()
        x = (w / 2) - (350 / 2)
        y = (h / 2) - (250 / 2)

        self.SetPosition((x,y))

    def OnClose(self, evt):
        self.Close()