import tensorflow as tf
import numpy as np

class LSTM(object):
    def __init__(self, learning_rate, embedding_vector_len, max_sentence_len, global_hidden_size,
                 num_classes, global_dropout, l2_regularization, is_training):

        """ Basic Configuration """
        self.learning_rate = learning_rate
        self.embedding_vector_len = embedding_vector_len
        self.max_sentence_len = max_sentence_len
        self.global_hidden_size = global_hidden_size
        self.num_classes = num_classes
        self.global_dropout = global_dropout
        self.l2_regularization = l2_regularization
        self.is_training = is_training

        """Placeholder Configuration"""
        self.X = tf.placeholder("float", [None, self.max_sentence_len, self.embedding_vector_len])
        self.Y = tf.placeholder("float", [None, self.num_classes])
        self.Sequence_len = tf.placeholder(tf.int32, [None])
        self.Aspect_vector = tf.placeholder("float", [None, self.embedding_vector_len])

        """for sess run"""
        self.prob = self.main_process(self.X, self.Sequence_len)
        self.loss_op = self.calculate_loss_op(self.Y, self.prob)
        self.train_op = self.training_op(self.loss_op)

        self.correct_pred = tf.equal(tf.argmax(tf.nn.softmax(self.prob), 1), tf.argmax(self.Y, 1))
        self.accuracy = tf.reduce_mean(tf.cast(self.correct_pred, tf.float32))

        """ global_variables_initializer """
        self.init = tf.global_variables_initializer()

    def main_process(self, X, Sequence_len):
        with tf.variable_scope("main_process", initializer=tf.contrib.layers.xavier_initializer()):
            with tf.variable_scope('lstm', reuse=tf.AUTO_REUSE):
                _, state = tf.nn.dynamic_rnn(tf.nn.rnn_cell.BasicLSTMCell(self.global_hidden_size), X,
                                           dtype=np.float32, sequence_length=Sequence_len)
                print(state)
            with tf.variable_scope('dense', reuse=tf.AUTO_REUSE):
                output = tf.layers.dense(state[1], 3)
                print(output)
        return output

    def calculate_loss_op(self, Y, prob):
        l2 = self.l2_regularization
        loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=tf.cast(Y, tf.float32), logits=prob))
        for v in tf.trainable_variables():
            if 'bias' not in v.name.lower():
                print("adding l2 regularization", v.name.lower())
                loss += l2 * tf.nn.l2_loss(v)
        return loss

    def training_op(self, loss_value):
        max_grad_val = 5.0
        """Calculate and apply gradients"""
        opt = tf.train.AdamOptimizer(learning_rate=self.learning_rate)
        gvs = opt.compute_gradients(loss_value)

        train_output = opt.apply_gradients(gvs)

        return train_output


#!/usr/bin/python3

from collections import defaultdict
from datetime import timedelta
from flask import (Flask, request, render_template,
                   session, redirect, url_for, escape)
import glob
import json
import logging
from logging import handlers
import os
import requests
import uuid
from waitress import serve

from bin.view_modifiers import response

def jawa_logger():
    global logger
    log_file = os.path.abspath(os.path.join(os.path.dirname(__file__), 'data', 'jawa.log'))
    logging.basicConfig(level=logging.INFO)
    logger = logging.getLogger('jawa')
    logger.setLevel(logging.INFO)
    handler = handlers.RotatingFileHandler(log_file, maxBytes=(1048576 * 100), backupCount=10)
    handler.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    handler.setFormatter(formatter)

    if logger.hasHandlers():
        logger.handlers.clear()
    logger.addHandler(handler)
    return logger

# Flask logging
jawa_logger()
error_message = ""
verify_ssl = True  # Enables Jamf Pro SSL certificate verification

# Initiate Flask
app = Flask(__name__)

# Session heartbeat
@app.before_request
def func():
    session.modified = True

def main():
    base_dir = os.path.dirname(__file__)
    jawa_logger().info(f"JAWA initializing...\n Sandcrawler home:  {base_dir}")
    environment_setup(base_dir)
    register_blueprints()
    app.secret_key = str(uuid.uuid4())
    app.permanent_session_lifetime = timedelta(minutes=10)
    serve(app, url_scheme='https', host='0.0.0.0', port=8000, threads=15)

def environment_setup(project_dir):
    global webhooks_file, cron_file, server_json_file, scripts_directory
    webhooks_file = os.path.abspath(os.path.join(project_dir, 'data', 'webhooks.json'))
    cron_file = os.path.abspath(os.path.join(project_dir, 'data', 'cron.json'))
    server_json_file = os.path.abspath(os.path.join(project_dir, 'data', 'server.json'))
    scripts_directory = os.path.abspath(os.path.join(project_dir, 'scripts'))
    jawa_logger().info(f"Detecting JAWA environment:\n"
                       f"Webhooks configuration file: {webhooks_file}\n"
                       f"Cron configuration file: {cron_file}\n"
                       f"Server configuration file: {server_json_file}\n"
                       f"Scripts directory: {scripts_directory}")

def register_blueprints():
    # JAWA Receiver
    from webhook import jawa_receiver
    app.register_blueprint(jawa_receiver.blueprint)
    # Jamf Pro Webhooks view
    from views import jamf_webhook
    app.register_blueprint(jamf_webhook.blueprint)
    # Okta Webhooks view
    from views.okta_webhook import blueprint
    app.register_blueprint(blueprint)
    # Create a new Cron Job
    from views.cron_views import blueprint
    app.register_blueprint(blueprint)
    # Log view
    from views import log_view
    app.register_blueprint(log_view.blueprint)
    # Resources (aka files) view
    from views import resource_view
    app.register_blueprint(resource_view.blueprint)
    # Custom Webhooks view
    from views import custom_webhook
    app.register_blueprint(custom_webhook.blueprint)
    # Webhooks Base view
    from views import webhook_view
    app.register_blueprint(webhook_view.blueprint)

# Server setup including making .json file necessary for webhooks

@app.route('/setup', methods=['GET', 'POST'])
def setup():
    if 'username' not in session:
        return redirect(url_for('logout', error_title="Session Timed Out", error_message="Please sign in again"))
    if request.method == 'POST':
        jawa_logger().debug(f"[{session.get('url')}] {session.get('username')} /setup - POST")
        server_url = request.form.get('address')
        if not server_url:
            return redirect(url_for('setup'))
        jps_url = request.form.get('jss-lock')
        jps2_check = request.form.get('alternate-jamf')
        jps_url2 = request.form.get('alternate')
        jawa_logger().info(f"{session.get('username')} made JAWA Setup Changes\n"
                           f"JAWA URL: {server_url}\n"
                           f"Primary JPS: {jps_url}\n"
                           f"Alternate JPS: {jps_url2}\n"
                           f"Alternate enabled?: {jps2_check}")
        new_json = {}
        if server_url != '':
            new_json['jawa_address'] = server_url
        # print(server_url)
        if jps_url != '':
            new_json['jps_url'] = jps_url
        # print(new_json)
        if not os.path.isfile(server_json_file):
            with open(server_json_file, "w") as outfile:
                server_json = {'jawa_address': server_url, 'jps_url': jps_url, 'alternate_jps': jps_url2}
                json.dump(server_json, outfile)
        elif os.path.isfile(server_json_file):
            with open(server_json_file, "w") as outfile:
                server_json = {'jawa_address': server_url, 'jps_url': jps_url, 'alternate_jps': jps_url2}
                json.dump(server_json, outfile)
            with open(server_json_file, "r") as fin:
                data = json.load(fin)
            data.update(new_json)
            with open(server_json_file, "w+") as outfile:
                json.dump(data, outfile)

        return render_template('success.html',
                               webhooks="success",
                               success_msg="JAWA Setup Complete!",
                               username=str(escape(session['username'])))
    else:
        jawa_logger().debug(f"[{session.get('url')}] {session.get('username')} - /setup - GET")
        if not os.path.isfile(server_json_file):
            with open(server_json_file, "w") as outfile:
                server_json = {'jawa_address': '', 'jps_url': '', 'alternate_jps': ''}
                json.dump(server_json, outfile)
        with open(server_json_file, "r") as fin:
            server_json = json.load(fin)
        jps_url2 = server_json.get('alternate_jps')
        if jps_url2 == str(escape(session['url'])):
            primary_jps = server_json['jps_url']
        else:
            primary_jps = str(escape(session['url']))
        jawa_url = server_json.get('jawa_address')
        return render_template('setup/setup.html',
                               login="false", jps_url=primary_jps, jps_url2=jps_url2,
                               jawa_url=jawa_url, username=session.get('username'))

@app.route("/cleanup", methods=['GET', 'POST'])
@response(template_file="setup/cleanup.html")
def cleanup():
    if 'username' not in session:
        return redirect(url_for('logout', error_title="Session Timed Out", error_message="Please sign in again"))
    if request.method != 'POST':
        return {"username": session.get('username'), "scripts_dir": scripts_directory}
    logger.info(f"[{session.get('url')}] {session.get('username')} is cleaning up scripts...")
    owd = os.getcwd()
    if not os.path.isdir(scripts_directory):
        os.mkdir(scripts_directory)
    os.chdir(scripts_directory)
    for file in glob.glob("*.old"):
        logger.info(f"[{session.get('url')}] {session.get('username')} removed the script {file}...")
        os.remove(file)
    os.chdir(owd)
    return redirect(url_for('success'))

@app.route('/login', methods=['GET', 'POST'])
def login():
    if request.method == 'POST':
        if os.path.isfile(server_json_file):
            with open(server_json_file) as json_file:
                server_json = json.load(json_file)
            if request.form.get('active_url'):
                session['url'] = str(request.form.get('active_url'))
            elif server_json.get('jps_url', 0):
                if server_json['jps_url'] is not None and len(server_json['jps_url']) != 0:
                    session['url'] = str(server_json['jps_url'])
            elif request.form.get('url')[-1:] == "/":
                session['url'] = str(request.form.get('url')).rstrip(request.form.get('url')[-1])
            else:
                session['url'] = request.form['url']

        elif request.form.get('active_url')[-1:] == "/":
            session['url'] = str(request.form.get('url')).rstrip(request.form.get('url')[-1])
        else:
            session['url'] = request.form['url']
        session['username'] = request.form['username']
        session['password'] = request.form['password']

        jawa_logger().info(f"[{session.get('url')}] Attempting login for: {session.get('username')}")

        if request.form['password'] == "":
            return redirect(url_for('logout', error_title="Authentication error", error_message="Passwords can't be blank"))
        try:
            resp = requests.get(
                session['url'] + '/JSSResource/activationcode',
                auth=(session['username'], session['password']),
                headers={'Accept': 'application/json'},
                verify=verify_ssl)

            resp.raise_for_status()

        except requests.exceptions.HTTPError as err:
            jawa_logger().info(f"Error occurred: {err}")
            return redirect(url_for('logout', error_title="HTTP Error", error_message=err))
        except requests.exceptions.ConnectTimeout as err:
            jawa_logger().info(f"Error occurred: {err}")
            return redirect(url_for('logout', error_title="Connection Timeout", error_message=err))
        except requests.exceptions.ConnectionError as err:
            jawa_logger().info(f"Error occurred: {err}")
            return redirect(url_for('logout', error_title="HTTP Error", error_message=err))

        jawa_logger().info(
            f"[{session.get('url')}] Logging In: " + str(escape(session['username'])))

        return redirect(url_for('dashboard'))

    if 'username' not in session:
        return redirect(url_for('logout', error_title="Session Timed Out", error_message="Please sign in again"))

    return redirect(url_for('dashboard'))

@app.route('/home.html')
def index():
    return load_home()

def load_home(error_title="", error_message=""):
    if 'username' in session:
        return redirect(url_for('dashboard'))
    if not os.path.isfile(server_json_file):
        with open(server_json_file, "w") as fout:
            json.dump({}, fout)

    with open(server_json_file, "r") as fin:
        server_json = json.load(fin)
    if not server_json:
        return render_template('home.html')
    brand = server_json.get("brand")

    if (
            'jps_url' not in server_json
            or server_json['jps_url'] is None
            or len(server_json['jps_url']) == 0
    ):
        return render_template('home.html', app_name=brand)
    if 'alternate_jps' not in server_json:
        return render_template('home.html', app_name=brand, error_title=error_title, error_message=error_message)

    if server_json['alternate_jps'] != "":
        return render_template('home.html',
                               jps_url=server_json['jps_url'],
                               jps_url2=server_json['alternate_jps'],
                               welcome="true", jsslock="true", app_name=brand, error_title=error_title,
                               error_message=error_message)

    session['url'] = server_json['jps_url']
    return render_template('home.html',
                           jps_url=str(escape(session['url'])),
                           welcome="true", jsslock="true", app_name=brand, error_title=error_title,
                           error_message=error_message)

@app.route("/")
def home():
    return load_home()

@app.route("/dashboard")
def dashboard():
    if 'username' not in session:
        return redirect(url_for('logout', error_title="Session Timed Out", error_message="Please sign in again"))
    jawa_logger().info(f"[{session.get('url')}] {session.get('username')} rendering /dashboard.")
    with open(webhooks_file) as webhook_json:
        webhooks_installed = json.load(webhook_json)
    jamf_pro_webhooks = []
    okta_webhooks = []
    custom_webhooks = []
    for each_webhook in webhooks_installed:
        data = defaultdict(lambda: "MISSING", each_webhook)
        tag = data['tag']
        if tag == "jamfpro":
            jamf_pro_webhooks.append(each_webhook)
        elif tag == "okta":
            okta_webhooks.append(each_webhook)
        elif tag == "custom":
            custom_webhooks.append(each_webhook)

    data = []

    if not os.path.isfile(cron_file):
        with open(cron_file, "w") as outfile:
            json.dump(data, outfile)

    with open(cron_file, "r") as cron_json:
        try:
            cron_list = json.load(cron_json)
        except json.decoder.JSONDecodeError as err:
            with open(cron_file, "w") as cron_json:
                cron_list = []
                json.dump(cron_list, cron_json, indent=4)

    cron_json = []
    for cron in cron_list:
        script = cron['script'].rsplit('/', 1)
        cron_json.append({"name": cron['name'],
                          "frequency": cron['frequency'],
                          "script": script[1],
                          "description": cron['description']})

    webhook_url = session['url']

    if not cron_json:
        cron_json = ''
    jawa_logger().info(f"Total webhooks managed by JAWA: {len(webhooks_installed)}")
    if webhook_json == cron_json:
        return redirect(url_for('first_automation'))
    return render_template(
        'dashboard.html',
        webhook_url=webhook_url,
        jamfpro_list=jamf_pro_webhooks,
        url=session.get('url'),
        cron_list=cron_json,
        okta_list=okta_webhooks,
        custom_list=custom_webhooks,
        total_webhooks=len(webhooks_installed),
        total_cron=len(cron_json),
        login="true",
        username=str(escape(session['username'])))

@app.route('/success', methods=['GET', 'POST'])
def success():
    if 'username' not in session:
        jawa_logger().info("No user logged in - returning to login page.")
        return redirect(url_for('logout', error_title="Session Timed Out", error_message="Please sign in again"))
    return render_template(
        'success.html',
        success_msg="",
        login="true",
        username=str(escape(session['username'])))

@app.route('/error', methods=['GET', 'POST'])
def error():
    error_title = request.args.get('error')
    error_message = request.args.get('error_message')
    if 'username' not in session:
        return redirect(url_for('logout'))
    jawa_logger().info(
        f"[{session.get('url')}] {session.get('username').title()} was a victim of a series of accidents, as are we all. (/error)")
    return render_template('error.html', username=session.get('username'), error_message=error_title,
                           error=error_message)

@app.errorhandler(404)
def page_not_found(error):
    if 'username' in session:
        jawa_logger().info(
            f"[{session.get('url')}] {session.get('username')} wandered off course  ({request.path}) - redirecting to /dashboard.")
        return redirect(url_for('dashboard'))
    jawa_logger().info(
        f"An invalid path ({request.path}) was provided and no user is logged in.  Returning login page.")
    return load_home()

@app.route('/logout')
def logout():
    error_title = request.args.get('error_title')
    error_message = request.args.get('error_message')
    if session.get('username'):
        logger.info("Logging Out: " + str(escape(session['username'])))
        session.pop('username', None)
    return load_home(error_title, error_message)

if __name__ == '__main__':
    main()


from setuptools import setup, find_packages
import sys, os

version = '0.4.7'

def read(*rnames):
    return open(os.path.join(os.path.dirname(__file__), *rnames)).read()

long_description = (
    read('README.txt')
    + '\n' +
    'Change history\n'
    '==============\n'
    + '\n' +
    read('CHANGES.txt')
    + '\n' +
    'Detailed Documentation\n'
    '======================\n'
    + '\n' +
    read('src', 'collective', 'plone', 'gsxml', 'README.rst')
    + '\n' +
    'Contributors\n'
    '============\n'
    + '\n' +
    read('CONTRIBUTORS.txt')
    + '\n' +
    'Download\n'
    '========\n'
    )

setup(name='collective.plone.gsxml',
      version=version,
      description="A package for importing and exporting content from Plone",
      long_description=long_description,
      # Get more strings from http://www.python.org/pypi?%3Aaction=list_classifiers
      classifiers=[
        "Framework :: Zope2",
        "Framework :: Zope3",
        "Programming Language :: Python",
        "Topic :: Software Development :: Libraries :: Python Modules",
        ],
      keywords='Plone import export XML',
      author='Stefan Eletzhofer, Ramon Bartl',
      author_email='stefan.eletzhofer@inquant.de',
      url='https://svn.plone.org/svn/collective/gsxml/releases/0.4.7',
      license='GPL',
      packages=find_packages('src'),
      package_dir = {'': 'src'},
      namespace_packages=['collective', 'collective.plone'],
      include_package_data=True,
      zip_safe=False,
      test_suite = 'collective.plone.gsxml.tests',
      extras_require = dict(
        test = [
            'zope.app.testing',
            'zope.interface',
            ],
        ),
      install_requires=[
        'setuptools',
      ],
      entry_points="""
      # -*- Entry points: -*-
      """,
      )


# -*- coding: utf-8 -*-

import re
import sys
from itertools import repeat
import jieba
import jieba.posseg as pseg
import pandas as pd

from keras.preprocessing.sequence import pad_sequences
from keras.utils import to_categorical
from keras.models import Model, Input, load_model
from keras.callbacks import EarlyStopping, ModelCheckpoint, TensorBoard
from keras.layers import LSTM, Embedding, Dense, TimeDistributed, Dropout, Bidirectional, concatenate
from keras.optimizers import Adagrad, Adadelta, RMSprop

from keras_contrib.layers import CRF
from keras.engine.topology import Layer

import numpy as np
from keras.callbacks import Callback
from sklearn.metrics import confusion_matrix, f1_score, precision_score, recall_score

import requests
from flask import Flask

import matplotlib.pyplot as plt

jieba.initialize()
# jieba.load_userdict('/media/zhipeng/OS/Document/intellij/twarp-kg/data/jieba-company-dict.txt')
# jieba.load_userdict('/media/zhipeng/OS/Document/intellij/twarp-kg/data/jieba-company-dict.txt')
jieba.load_userdict('C:/Document/intellij/twarp-kg/data/jieba-company-dict.txt')

app = Flask(__name__)

@app.route("/")
def hello_world():
    return 'Hello, World!'

def zero_digits(s):
    """
    Replace every digit in a string by a zero.
    """
    return re.sub('\d', '0', s)

def iob2(tags):
    """
    Check that tags have a valid IOB format.
    Tags in IOB1 format are converted to IOB2.
    """
    for i, tag in enumerate(tags):
        if tag == 'O':
            continue
        split = tag.split('-')
        if len(split) != 2 or split[0] not in ['I', 'B']:
            return False
        if split[0] == 'B':
            continue
        elif i == 0 or tags[i - 1] == 'O':  # conversion IOB1 to IOB2
            tags[i] = 'B' + tag[1:]
        elif tags[i - 1][1:] == tag[1:]:
            continue
        else:  # conversion IOB1 to IOB2
            tags[i] = 'B' + tag[1:]
    return True

def iob_iobes(tags):
    """
    IOB -> IOBES
    """
    new_tags = []
    for i, tag in enumerate(tags):
        if tag == 'O':
            new_tags.append(tag)
        elif tag.split('-')[0] == 'B':
            if i + 1 != len(tags) and \
                    tags[i + 1].split('-')[0] == 'I':
                new_tags.append(tag)
            else:
                new_tags.append(tag.replace('B-', 'S-'))
        elif tag.split('-')[0] == 'I':
            if i + 1 < len(tags) and \
                    tags[i + 1].split('-')[0] == 'I':
                new_tags.append(tag)
            else:
                new_tags.append(tag.replace('I-', 'E-'))
        else:
            raise Exception('Invalid IOB format!')
    return new_tags

def iobes_iob(tags):
    """
    IOBES -> IOB
    """
    new_tags = []
    for i, tag in enumerate(tags):
        if tag.split('-')[0] == 'B':
            new_tags.append(tag)
        elif tag.split('-')[0] == 'I':
            new_tags.append(tag)
        elif tag.split('-')[0] == 'S':
            new_tags.append(tag.replace('S-', 'B-'))
        elif tag.split('-')[0] == 'E':
            new_tags.append(tag.replace('E-', 'I-'))
        elif tag.split('-')[0] == 'O':
            new_tags.append(tag)
        else:
            raise Exception('Invalid format!')
    return new_tags

def update_tag_scheme(sentences, tag_scheme):
    """
    Check and update sentences tagging scheme to IOB2.
    Only IOB1 and IOB2 schemes are accepted.
    """
    for i, s in enumerate(sentences):
        tags = [w[-1] for w in s]
        # Check that tags are given in the IOB format
        if not iob2(tags):
            s_str = '\n'.join(' '.join(w) for w in s)
            raise Exception('Sentences should be given in IOB format! ' +
                            'Please check sentence %i:\n%s' % (i, s_str))
        if tag_scheme == 'iob':
            # If format was IOB1, we convert to IOB2
            for word, new_tag in zip(s, tags):
                word[-1] = new_tag
        elif tag_scheme == 'iobes':
            new_tags = iob_iobes(tags)
            for word, new_tag in zip(s, new_tags):
                word[-1] = new_tag
        else:
            raise Exception('Unknown tagging scheme!')

def word2bmes(word):
    if len(word) == 1:
        for _ in word:
            yield 'S'
    else:
        for i in range(len(word)):
            if i == 0:
                yield 'B'
            elif i == len(word) - 1:
                yield 'E'
            else:
                yield 'M'

class Token(object):

    def __init__(self, word, flag):
        self.word = word
        self.flag = flag

def tokenize(text, use_jieba=True):
    if use_jieba:
        return [token for token in pseg.cut(text)]
    url = 'http://localhost:8089/nlp/tokenize'
    response = requests.post(url, data=text.encode('u8'), headers={"Content-Type": "text/plain;charset=UTF-8"})
    response = response.json()
    return [Token(t[0], t[1]) for t in zip(response['seg'], response['pos'])]

# return sentences with shape (n, len, 2)
def read_ner_data(path, zeros=False):
    sentences = []
    with open(path, "r", encoding='u8') as f:
        sentence = []
        n = 0
        for raw_line in f:
            n += 1
            line = zero_digits(raw_line.strip()) if zeros else raw_line.strip()
            if len(line) == 0:
                if len(sentence) > 0:
                    sentences.append(sentence)
                sentence = []
            else:
                splits = line.split()
                if len(splits) == 1 and line[0] in ["I", "O", "B", "E", "S"]:
                    sentence.append([" ", line])
                else:
                    assert len(splits) == 2, "line %d: %s, %d, %d" % (n, raw_line.rstrip(), len(splits), len(line))
                    sentence.append(splits)
    return sentences

# return sentences in n * 4 * sen_len
def seg_ner_sentences(data):
    sentences = []
    for sentence in data:
        words = [s[0] for s in sentence]
        ner_tags = [s[1] for s in sentence]
        # tokens = [token for token in pseg.cut(''.join(words))]
        tokens = tokenize(''.join(words))
        # tags = [token.tag for _ in range(len(token.word)) for token in tokens]
        tags = [tag for t in tokens for tag in repeat(t.flag, len(t.word))]
        segs = [seg for token in tokens for seg in word2bmes(token.word)]
        sentences.append([words, segs, tags, ner_tags])
        # row = [words, segs, tags, ner_tags]
        # sentences.append([[row[i][j] for j in range(len(words))] for i in range(len(row))])
        # sentences.append([[words[i], segs[i], tags[i], ner_tags[i]] for i in range(len(sentence))])
    return sentences

class NERTransformer(object):

    def __init__(self, ner_train_file, max_len=70, zeros=False, use_iobes=True, use_bmes=True):
        sentences = read_ner_data(ner_train_file, zeros)
        self.original_sentences = [''.join([t[0] for t in s]) for s in sentences]

        self.ner_schema = 'iobes' if use_iobes else 'iob'
        update_tag_scheme(sentences, self.ner_schema)
        sentences = seg_ner_sentences(sentences)

        self.sentences = sentences
        self.max_len = max_len

        self.words = list(set((w for s in sentences for w in s[0])))
        # words = list(set((w[0] for s in sentences for w in s)))
        self.words.append('<UNK>')
        self.n_words = len(self.words)

        self.pos_schema = 'BMES' if use_bmes else 'IOB'
        self.seg_tags = [c for c in self.pos_schema]
        self.n_seg_tags = len(self.seg_tags)

        self.pos_tags = list(set((w for s in sentences for w in s[2])))
        self.pos_tags.append('<UNK>')
        self.n_pos_tags = len(self.pos_tags)

        self.ner_tags = list(set((w for s in sentences for w in s[3])))
        self.n_ner_tags = len(self.ner_tags)
        print("n_words: %d, n_pos_tags: %d, n_tags: %d" % (self.n_words, self.n_pos_tags, self.n_ner_tags))

        self.char2idx = {w: i + 1 for i, w in enumerate(self.words)}
        self.seg2idx = {w: i + 1 for i, w in enumerate(self.seg_tags)}
        self.pos2idx = {w: i + 1 for i, w in enumerate(self.pos_tags)}
        self.ner2idx = {t: i for i, t in enumerate(self.ner_tags)}

        self.idx2char = {v: k for k, v in self.char2idx.items()}
        self.idx2seg = {v: k for k, v in self.seg2idx.items()}
        self.idx2pos = {v: k for k, v in self.pos2idx.items()}
        self.idx2ner = {v: k for k, v in self.ner2idx.items()}

    def get_train_data(self):
        X_char = [[self.char2idx[w] for w in s[0]] for s in self.sentences]
        X_seg = [[self.seg2idx[w] for w in s[1]] for s in self.sentences]
        X_pos = [[self.pos2idx[w] for w in s[2]] for s in self.sentences]
        X_char = pad_sequences(maxlen=self.max_len, sequences=X_char, padding="post", value=0)
        X_seg = pad_sequences(maxlen=self.max_len, sequences=X_seg, padding="post", value=0)
        X_pos = pad_sequences(maxlen=self.max_len, sequences=X_pos, padding="post", value=0)

        # X = np.reshape(X, (3, len(X), -1))
        # X = [pad_sequences(maxlen=max_len, sequences=seq, padding="post", value=0) for seq in X]
        # X = np.reshape(X, (-1, 3, max_len))
        # X = pad_sequences(maxlen=max_len, sequences=X, padding="post", value=0)
        y = [[self.ner2idx[w] for w in s[3]] for s in self.sentences]
        y = pad_sequences(maxlen=self.max_len, sequences=y, padding="post", value=self.ner2idx['O'])
        # For training the network we also need to change the labels y to categorial.
        y = [to_categorical(i, num_classes=self.n_ner_tags) for i in y]

        return [X_char, X_seg, X_pos], np.array(y)

    def transform_sentences_as_input(self, sentences):
        char_input = []
        seg_input = []
        pos_input = []
        # sentences = [sentences] if isinstance(sentences, str) else sentences
        for sentence in sentences:
            # char_input.append([self.char2idx.get(w, self.n_words) for w in sentence])
            # tokens = [token for token in pseg.cut(sentence)]
            tokens = tokenize(sentence)
            transformed_sentence = ''.join([token.word for token in tokens])
            char_input.append([self.char2idx.get(w, self.n_words) for w in transformed_sentence])
            seg_input.append([self.seg2idx[seg] for token in tokens for seg in word2bmes(token.word)])
            pos_input.append(
                [self.pos2idx.get(pos, self.n_pos_tags) for t in tokens for pos in repeat(t.flag, len(t.word))])

        char_input = pad_sequences(maxlen=self.max_len, sequences=char_input, padding="post", value=0)
        seg_input = pad_sequences(maxlen=self.max_len, sequences=seg_input, padding="post", value=0)
        pos_input = pad_sequences(maxlen=self.max_len, sequences=pos_input, padding="post", value=0)

        return [char_input, seg_input, pos_input]

    def parse_prediction(self, x, y):
        # assert len(x[0]) == len(y), 'x and y must have the same length, len(x)=%d, len(y)=%d' % (len(x[0]), len(y))
        segments = []
        ner_tags = []
        entities = []
        for sentence, prediction in zip(x, y):
            # sentence = ''.join([self.idx2char[idx] for idx in x[0][i] if idx > 0])
            # tokens = [token.word + '/' + token.flag for token in pseg.cut(sentence)]
            tokens = tokenize(sentence)
            sentence = ''.join([token.word for token in tokens])
            prediction = [self.idx2ner[idx] for idx in np.argmax(prediction, axis=-1)][:len(sentence)]
            segments.append(' '.join([token.word + '/' + token.flag for token in tokens]))
            ner_tags.append(' '.join([c + '/' + t for c, t in zip(sentence, prediction)]))

            i = 0
            assert self.ner_schema == 'iobes', "ner tag schema currently only supports iobes"
            sentence_entities = []
            while i < len(prediction):  # prediction length may be shorter than sentence due to max_len padding
                tag = prediction[i]
                i += 1
                if tag.startswith('S'):
                    sentence_entities.append({'start': i - 1, 'end': i, 'ner': sentence[i - 1], 'tag': tag[2:]})
                elif tag.startswith('B'):
                    entity = {'start': i - 1, 'tag': tag[2:]}
                    while i < len(prediction) and prediction[i][0] == 'I' and prediction[i].endswith(entity['tag']):
                        i += 1
                    if i < len(prediction) and prediction[i] == 'E-' + entity['tag']:
                        i += 1
                        entity['end'] = i
                        entity['word'] = sentence[entity['start']:entity['end']]
                        sentence_entities.append(entity)
                # elif tag.startswith('I') or tag.startswith('E'):
                #     i += 1
                # else:
                #     i += 1

            entities.append(sentence_entities)

        return x, segments, ner_tags, entities

    def recognize_entities(self, model, sentences):
        if isinstance(sentences, (str,)):
            sentences = [sentences]
        y = model.predict(self.transform_sentences_as_input(sentences))
        return self.parse_prediction(sentences, y)

class TagRegularizer(Layer):

    def __init__(self, output_dim, **kwargs):
        self.output_dim = output_dim
        super(TagRegularizer, self).__init__(**kwargs)

    def build(self, input_shape):
        # Create a trainable weight variable for this layer.
        self.W = self.add_weight(shape=(input_shape[1], self.output_dim),
                                 initializer='uniform',
                                 trainable=True)
        super(TagRegularizer, self).build(input_shape)  # Be sure to call this somewhere!

    def call(self, x, mask=None):
        from keras import backend as K
        return K.dot(x, self.W)

    def get_output_shape_for(self, input_shape):
        return input_shape[0], self.output_dim

class Metrics(Callback):

    def __init__(self):
        super().__init__()
        self.val_f1s = []
        self.val_recalls = []
        self.val_precisions = []

    def on_train_begin(self, logs={}):
        self.val_f1s = []
        self.val_recalls = []
        self.val_precisions = []

    def on_epoch_end(self, epoch, logs={}):
        val_predict = (np.asarray(self.model.predict(self.model.validation_data[0]))).round()
        val_targ = self.model.validation_data[1]
        f1 = f1_score(val_targ, val_predict)
        recall = recall_score(val_targ, val_predict)
        precision = precision_score(val_targ, val_predict)
        self.val_f1s.append(f1)
        self.val_recalls.append(recall)
        self.val_precisions.append(precision)
        print("val_f1: % f — val_precision: % f — val_recall % f" % (f1, precision, recall))

epochs = 10
batch_size = 64
max_len = 70
char_emb_size = 80
seg_emb_size = 8
pos_emb_size = 12

zeros = False
use_bmes = True
use_iobes = True
use_pretrained_embedding = True

# train_file = '/media/zhipeng/OS/Document/intellij/twarp-kg/data/industry-ner-train-argumented.txt'
train_file = '/Document/intellij/twarp-kg/data/industry-ner-train-argumented.txt'
# train_file = '../data/industry-ner-train-argumented.txt'
ner = NERTransformer(train_file, max_len, zeros, use_iobes, use_bmes)
X, y = ner.get_train_data()

char_input = Input(shape=(max_len,), dtype='int32', name='char_input')
seg_input = Input(shape=(max_len,), dtype='int32', name='seg_input')
pos_input = Input(shape=(max_len,), dtype='int32', name='pos_input')

# char_embedding = Embedding(n_words + 1, char_emb_size, input_length=max_len, mask_zero=True)(input[:, 0, :])
char_embedding = Embedding(ner.n_words + 1, char_emb_size, input_length=ner.max_len, mask_zero=True)(char_input)
# seg_embedding = Embedding(n_seg_tags + 1, seg_emb_size, input_length=max_len, mask_zero=True)(input[:, 1, :])
seg_embedding = Embedding(ner.n_seg_tags + 1, seg_emb_size, input_length=max_len, mask_zero=True)(seg_input)
# pos_embedding = Embedding(n_pos_tags + 1, pos_emb_size, input_length=max_len, mask_zero=True)(input[:, 2, :])
pos_embedding = Embedding(ner.n_pos_tags + 1, pos_emb_size, input_length=max_len, mask_zero=True)(pos_input)

embedding = concatenate([char_embedding, seg_embedding, pos_embedding])

model = Dropout(rate=0.3)(embedding)
model = Bidirectional(LSTM(units=70, return_sequences=True, recurrent_dropout=0.1))(model)  # variational biLSTM
model = TimeDistributed(Dense(50, activation="relu"))(model)  # a dense layer as suggested by neuralNer

crf = CRF(units=ner.n_ner_tags, learn_mode='join', test_mode='viterbi')  # CRF layer
out = crf(model)  # output

optimizer = RMSprop(lr=0.01, epsilon=1e-06, clipvalue=3)
# optimizer = Adadelta(lr=1.0, epsilon=1e-06, clipvalue=3)
# optimizer = RMSprop(lr=0.001, epsilon=1e-06, clipvalue=3)
model = Model([char_input, seg_input, pos_input], out)
model.compile(optimizer=optimizer, loss=crf.loss_function, metrics=[crf.accuracy])
model.summary()

model_path = '../data/model/keras-ner.h5'
weights_path = '../data/model/keras-ner-weights.h5'
checkpoints = '../data/model/keras-ner-checkpoints.h5'

if len(sys.argv) > 1 and sys.argv[1] == 'test':
    from keras_contrib import *
    # model.load_weights(weights_path)
    # https://github.com/keras-team/keras-contrib/issues/125
    # model = load_model(model_path, custom_objects={'CRF': CRF, 'loss': crf.loss_function})
    from keras_contrib.utils import save_load_utils
    save_load_utils.load_all_weights(model, weights_path, include_optimizer=True)
else:
    # Keep only a single checkpoint, the best over test accuracy. ['loss', 'acc', 'val_loss', 'val_acc']
    checkpoint = ModelCheckpoint(checkpoints, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
    # modelCheckpoints = ModelCheckpoint("checkpoints/HoboNet_{epoch:02d}-{val_loss:.2f}-{val_acc:.2f}.hdf5",
    #                                    monitor='val_loss', verbose=0, save_best_only=False,
    #                                    save_weights_only=False, mode='auto')
    tensorBoard = TensorBoard(log_dir='../data/log/keras-ner', histogram_freq=1, write_graph=True)
    early_stop = EarlyStopping(monitor='val_loss', min_delta=0, patience=3)

    history = model.fit(X, y, batch_size=64, epochs=10, validation_split=0.3, verbose=1,
                        callbacks=[checkpoint, early_stop, tensorBoard])

    # model.save_weights(weights_path)
    # model.save(model_path)
    from keras_contrib.utils import save_load_utils

    save_load_utils.save_all_weights(model, weights_path, include_optimizer=True)

    hist = pd.DataFrame(history.history)
    plt.style.use("ggplot")
    plt.figure(figsize=(6, 6))

    plt.plot(hist["acc"])
    plt.plot(hist["val_acc"])
    plt.show()  # will block the program

n_test = 50
X_test = [X[0][:n_test], X[1][:n_test], X[2][:n_test]]
sentences = ner.original_sentences[:n_test]
prediction = model.predict(X_test)

sentences, segments, ner_tags, entities = ner.parse_prediction(sentences, prediction)
for result in zip(sentences, segments, ner_tags, entities):
    print('\n'.join([str(e) for e in result]), end='\n\n')

text = '1.中小型锂电池龙头企业。2014年营业收入1417.72亿元，同比下降18.07%，主要原因是公司2013年6月处置铝加工业务导致报告期营业收入中未包括铝加工板块收入，以及本年产品销售价格降低和销售量减少;实现归属于上市公司股东净利润-162.17亿元，主要系产品售价下滑、大额长期资产计提减值准备、三项费用较高所致。2015年10月底公司完成对河南农化67%的股权收购，报告期合并报表，形成新的利润增长点(上半年河南农化贡献利润1379万元)，带动公司前三季度业绩提升。2015年第三方医学检测市场份额约为医学检验市场的3%，分级诊疗等政策利好下，第三方检验业务发展迅速。2016年1-8月，佳芃文化营业收入为613万元，净利润447.7万元。2016年公司在有色金属采选业务之外，大力发展金属供应链金融和产业链金融服务业务。GAMA预计2015年全球通用航空飞机交付价值将超过250亿美元，其中美国市场交出了一份较强的交付数据，整个通航飞机市场呈复苏的趋势。"一号文件"渐行渐近农业供给侧改革望成焦点"。一季报业绩稳健提升，定增收购正泰新能源获证监会受理:据公司2016年一季报显示，实现营业收入23.82亿元，同比增长5.31%，归母净利润3.24亿元，同比增长7.28%，经营活动产生的现金流量净额3.97亿元，现金流状况良好;近期，公司公告其拟发行股份购买正泰新能源(主营光伏电站开发建设、EPC和组件制造)100%股权并募集配套资金45亿元项目已获得证监会受理。三维激光切割是多轴机器人系统集成的具体应用，是替代传统工艺的新兴产业。与此同时，公司配套募投的“曲面显示用盖板玻璃生产项目”目前已形成500万片的年产能力，并为客户批量供货，为顺应OLED柔性显示屏趋势打下基础。且半导体设备技术壁垒较高，行业呈现高集中度特点，国际前十大半导体设备厂占市场绝对优势，主要集中于美国，日本及荷兰，而国内厂商处于起步阶段，产业相对薄弱。业内人士分析，根据工信部部署以及三大运营商的布局情况，预计在2017年或2018年展开5G网络建设'
sentences = re.compile('[。,，；]').split(text)
X_test = ner.transform_sentences_as_input(sentences)
prediction = model.predict(X_test)

sentences, segments, ner_tags, entities = ner.parse_prediction(sentences, prediction)
for result in zip(sentences, segments, ner_tags, entities):
    print('\n'.join([str(e) for e in result]), end='\n\n')

while True:
    sentence = input("请输入测试句子:")
    sentence = sentence.strip()
    if len(sentence) > 4:
        _, segments, ner_tags, entities = ner.recognize_entities(model, sentence)
        for result in zip(segments, ner_tags, entities):
            print('\n'.join([str(e) for e in result]), end='\n\n')
    else:
        print('sentence is too short')


#Author: Andrew Mohnkern
#Date: 7/2/19
#Class: CIS 220
#Project: Enigma Encryptor
#Filename: gui.py
#Desc: Provides functional GUI

import utilities
from appJar import gui

def run_gui():

	#static class to hold file paths
	class FilePaths():

		InFilePath = ""
		OutFilePath = ""

		def checkInput(): #returns false if input filepath is invalid, throws appropriate errorboxes
			InExt = FilePaths.InFilePath[-4:] #takes last four characters

			if FilePaths.InFilePath == "":
				appMain.errorBox("Error", "You have not selected a file to encrypt.")
				return False

			elif InExt != ".txt":
				appMain.errorBox("Error", "Your input file must be a text file.")
				FilePaths.InFilePath = ""
				return False

			else:
				return True

		def checkOutput(): #returns false if output filepath is invalid, throws appropriate errorBoxes
			OutExt = FilePaths.OutFilePath[-4:] #takes last four characters

			if FilePaths.OutFilePath == "":
				appMain.errorBox("Error", "You have not selected a file to output to.")
				return False

			elif OutExt != ".txt":
				#appMain.errorBox("Error", "Your output file must be a text file.")
				FilePaths.OutFilePath += ".txt"
				return False

			else:
				return True

	#Main window
	appMain = gui("Enigma", "800x600")
	appMain.setTitle("Enigma Machine")
	appMain.setIcon("assets/icon.gif") #does not work for some reason
	appMain.setBg("#6B7A8F")
	appMain.setFont(family="Verdana")

	#TOP
	appMain.startFrame("TOP", row=0, column=0, colspan=2) #centered top frame
	appMain.setBg("#6B7A8F")
	appMain.addImage("banner", "assets/banner.gif")

	appMain.stopFrame()

	#MIDDLE LEFT
	appMain.startFrame("MIDDLE_LEFT", row=1, column=0)

	appMain.addLabel("InputFile", "")
	appMain.addLabel("OutputFile", "")
	appMain.getLabelWidget("InputFile").config(font=("Verdana 14"))
	appMain.getLabelWidget("OutputFile").config(font=("Verdana 14"))
	appMain.setLabelAlign("InputFile", "left")
	appMain.setLabelAlign("OutputFile", "left")
	appMain.setLabelPadding("InputFile", [20, 0])
	appMain.setLabelPadding("OutputFile", [20, 0])

	#color
	appMain.setLabelFg("InputFile", "white")
	appMain.setLabelFg("OutputFile", "white")

	appMain.stopFrame()

	#MIDDLE RIGHT
	appMain.startFrame("MIDDLE_RIGHT", row=1, column=1)

	#Button funcs
	def selectIn(): #file selection in
		FilePaths.InFilePath = utilities.getTextFilePath()
		null = FilePaths.checkInput()
		appMain.setLabel("InputFile", FilePaths.InFilePath)

	def selectOut(): #file selection out
		FilePaths.OutFilePath = utilities.getSaveFilePath()
		null = FilePaths.checkOutput()
		appMain.setLabel("OutputFile", FilePaths.OutFilePath)

	#Selection buttons
	appMain.addButton("SelectIn", selectIn)
	appMain.addButton("SelectOut", selectOut)
	appMain.setButton("SelectIn", "Choose input file")
	appMain.setButton("SelectOut", "Choose output file")
	appMain.setButtonSticky("SelectIn", "")
	appMain.setButtonSticky("SelectOut", "")

	#Color
	appMain.setButtonFg("SelectIn", "white")
	appMain.setButtonBg("SelectIn", "#F7882F")
	appMain.setButtonFg("SelectOut", "white")
	appMain.setButtonBg("SelectOut", "#F7882F")

	appMain.stopFrame()

	#BOTTOM
	appMain.startFrame("BOTTOM", row=2, column=0, colspan=2) #centered bottom frame

	def encryptButton():
		if FilePaths.checkInput():
			if FilePaths.checkOutput(): #check in and out, just in case
				utilities.encryptTextArgs(FilePaths.InFilePath, FilePaths.OutFilePath)
				appMain.infoBox("Done", "Encryption complete")

	def decryptButton():
		if FilePaths.checkInput():
			if FilePaths.checkOutput(): #check in and out, just in case
				utilities.decryptTextArgs(FilePaths.InFilePath, FilePaths.OutFilePath)
				appMain.infoBox("Done", "Decryption complete")

	def mainButtons(button):
		#Encrypt
		if button == "Encrypt":
			encryptButton()

		#Decrypt
		elif button == "Decrypt":
			decryptButton()

		#Quit
		elif button == "Quit":
			appMain.stop()
			print("Application quit.")

		elif button == "Instructions":
			appMain.infoBox("Instructions", "Use the buttons on the right to select your input file and output destination. Then press the encrypt or decrypt button to create the output file.")

	buttonList = ["Encrypt", "Decrypt", "Instructions", "Quit"]

	appMain.addButtons(buttonList, mainButtons)

	#Color
	for i in buttonList:
		appMain.setButtonFg(i, "white")
		appMain.setButtonBg(i, "#F7882F")
		#appMain.setButtonWidth(i, 90) #This breaks everything why
		#appMain.setButtonHeight(i, 100)

	appMain.stopFrame()

	#MUST BE AT BOTTOM
	appMain.go()


from __future__ import absolute_import, division, print_function

import os
from tqdm import tqdm
import base64
import imageio
import IPython
import matplotlib
matplotlib.use('pdf')
import matplotlib.pyplot as plt
import gym
import tensorflow as tf
import gym_backcheetah

from absl import flags
from absl import app
tf.compat.v1.enable_v2_behavior()

from tf_agents.agents.ddpg import critic_network
from tf_agents.agents.sac import sac_agent
from tf_agents.drivers import dynamic_step_driver
from tf_agents.environments import suite_gym
from tf_agents.environments import tf_py_environment
from tf_agents.eval import metric_utils
from tf_agents.metrics import tf_metrics
from tf_agents.networks import actor_distribution_network
from tf_agents.networks import normal_projection_network
from tf_agents.policies import greedy_policy
from tf_agents.policies import random_tf_policy
from tf_agents.replay_buffers import tf_uniform_replay_buffer
from tf_agents.trajectories import trajectory
from tf_agents.utils import common

FLAGS = flags.FLAGS
flags.DEFINE_string('root_dir', './', 'root dir for output')
flags.DEFINE_string('env_name', 'Hopper-v2', 'Name of a mujoco env')
flags.DEFINE_float('lr', 3e-4, 'learning rate of neural networks')
flags.DEFINE_integer('num_iterations', 1000000, 'number of iterations')
flags.DEFINE_boolean('anneal_lr', False, 'whether or not to anneal learning rate')
flags.DEFINE_float('target_entropy', None, 'target entropy to learn to')

def main(_):
  env_name = FLAGS.env_name
  num_iterations = FLAGS.num_iterations # @param {type:"integer"}
  initial_collect_steps = 10000 # @param {type:"integer"}
  collect_steps_per_iteration = 1 # @param {type:"integer"}
  replay_buffer_capacity = 1000000 # @param {type:"integer"}

  root_dir = FLAGS.root_dir
  batch_size = 256 # @param {type:"integer"}

  critic_learning_rate = FLAGS.lr # @param {type:"number"}
  actor_learning_rate = FLAGS.lr # @param {type:"number"}
  alpha_learning_rate = FLAGS.lr # @param {type:"number"}
  anneal_lr = FLAGS.anneal_lr
  target_update_tau = 0.005 # @param {type:"number"}
  target_update_period = 1 # @param {type:"number"}
  gamma = 0.99
  reward_scale_factor = 1.0 # @param {type:"number"}
  gradient_clipping = True # @param

  actor_fc_layer_params = (256, 256)
  critic_joint_fc_layer_params = (256, 256)

  log_interval = 5000 # @param {type:"integer"}

  num_eval_episodes = 30 # @param {type:"integer"}
  eval_interval = 10000 # @param {type:"integer"}
  video_interval = 10000000
  plot_interval = 1000000
  train_checkpoint_interval = 10000000
  target_entropy = FLAGS.target_entropy

  def embed_mp4(filename):
    """Embeds an mp4 file in the notebook."""
    video = open(filename,'rb').read()
    b64 = base64.b64encode(video)
    tag = '''
    <video width="640" height="480" controls>
      <source src="data:video/mp4;base64,{0}" type="video/mp4">
    Your browser does not support the video tag.
    </video>'''.format(b64.decode())

    return IPython.display.HTML(tag)

  train_py_env = suite_gym.load(env_name)
  eval_py_env = suite_gym.load(env_name)
  train_env = tf_py_environment.TFPyEnvironment(train_py_env)
  eval_env = tf_py_environment.TFPyEnvironment(eval_py_env)

  observation_spec = train_env.observation_spec()
  action_spec = train_env.action_spec()
  critic_net = critic_network.CriticNetwork(
      (observation_spec, action_spec),
      observation_fc_layer_params=None,
      action_fc_layer_params=None,
      joint_fc_layer_params=critic_joint_fc_layer_params)

  def normal_projection_net(action_spec,init_means_output_factor=0.1):
    return normal_projection_network.NormalProjectionNetwork(
        action_spec,
        mean_transform=None,
        state_dependent_std=True,
        init_means_output_factor=init_means_output_factor,
        std_transform=sac_agent.std_clip_transform,
        scale_distribution=True)

  actor_net = actor_distribution_network.ActorDistributionNetwork(
      observation_spec,
      action_spec,
      fc_layer_params=actor_fc_layer_params,
      continuous_projection_net=normal_projection_net)

  global_step = tf.compat.v1.train.get_or_create_global_step()
  if anneal_lr:
    boundaries = [600000, 1000000]
    values = [3e-4, 9e-5, 5e-5]
    learning_rate = tf.compat.v1.train.piecewise_constant(global_step, boundaries, values)
  tf_agent = sac_agent.SacAgent(
      train_env.time_step_spec(),
      action_spec,
      actor_network=actor_net,
      critic_network=critic_net,
      actor_optimizer=tf.compat.v1.train.AdamOptimizer(
          learning_rate=actor_learning_rate),
      critic_optimizer=tf.compat.v1.train.AdamOptimizer(
          learning_rate=critic_learning_rate),
      alpha_optimizer=tf.compat.v1.train.AdamOptimizer(
          learning_rate=alpha_learning_rate),
      target_update_tau=target_update_tau,
      target_update_period=target_update_period,
      td_errors_loss_fn=tf.compat.v1.losses.mean_squared_error,
      gamma=gamma,
      reward_scale_factor=reward_scale_factor,
      gradient_clipping=gradient_clipping,
      train_step_counter=global_step,
      target_entropy=target_entropy)
  tf_agent.initialize()

  eval_policy = greedy_policy.GreedyPolicy(tf_agent.policy)
  collect_policy = tf_agent.collect_policy

  def compute_avg_return(environment, policy, num_episodes=5):

    total_return = 0.0
    for _ in range(num_episodes):

      time_step = environment.reset()
      episode_return = 0.0

      while not time_step.is_last():
        action_step = policy.action(time_step)
        time_step = environment.step(action_step.action)
        episode_return += time_step.reward
      total_return += episode_return
    avg_return = total_return / num_episodes
    return avg_return.numpy()[0]

  replay_buffer = tf_uniform_replay_buffer.TFUniformReplayBuffer(
      data_spec=tf_agent.collect_data_spec,
      batch_size=train_env.batch_size,
      max_length=replay_buffer_capacity)

  initial_collect_driver = dynamic_step_driver.DynamicStepDriver(
          train_env,
          collect_policy,
          observers=[replay_buffer.add_batch],
          num_steps=initial_collect_steps)
  print("Running initial collection policy...")
  initial_collect_driver.run()

  # Dataset generates trajectories with shape [Bx2x...]
  dataset = replay_buffer.as_dataset(
      num_parallel_calls=3, sample_batch_size=batch_size, num_steps=2).prefetch(3)

  iterator = iter(dataset)

  collect_driver = dynamic_step_driver.DynamicStepDriver(
      train_env,
      collect_policy,
      observers=[replay_buffer.add_batch],
      num_steps=collect_steps_per_iteration)

  # #@test {"skip": true}
  # %%time

  # (Optional) Optimize by wrapping some of the code in a graph using TF function.
  tf_agent.train = common.function(tf_agent.train)
  collect_driver.run = common.function(collect_driver.run)

  # Reset the train step
  tf_agent.train_step_counter.assign(0)

  # Set up checkpointing of tf agents model
  train_metrics = [
        tf_metrics.NumberOfEpisodes(),
        tf_metrics.EnvironmentSteps(),
        tf_metrics.AverageReturnMetric(),
        tf_metrics.AverageEpisodeLengthMetric(),
    ]
  train_dir = os.path.join(root_dir, 'train_ckpt')
  train_checkpointer = common.Checkpointer(
        ckpt_dir=train_dir,
        agent=tf_agent,
        global_step=global_step,
        metrics=metric_utils.MetricsGroup(train_metrics, 'train_metrics'))
  train_checkpointer.initialize_or_restore()

  # Evaluate the agent's policy once before training.
  print("Initial policy evaluation...")
  avg_return = compute_avg_return(eval_env, eval_policy, num_eval_episodes)
  returns = [avg_return]

  returnsCache = []
  print("Training...")
  for _ in tqdm(range(num_iterations)):
    # Collect a few steps using collect_policy and save to the replay buffer.
    for _ in range(collect_steps_per_iteration):
      collect_driver.run()

    # Sample a batch of data from the buffer and update the agent's network.
    experience, unused_info = next(iterator)
    train_loss = tf_agent.train(experience)

    step = tf_agent.train_step_counter.numpy()
    if step % log_interval == 0:
      print('step = {0}: loss = {1}'.format(step, train_loss.loss))

    if step % eval_interval == 0:
      avg_return = compute_avg_return(eval_env, eval_policy, num_eval_episodes)
      print('step = {0}: Average Return = {1}'.format(step, avg_return))
      returnsCache.append((step, avg_return))

    if step % train_checkpoint_interval == 0:
      print("Saving model checkpoint...")
      train_checkpointer.save(global_step=global_step)

    if step % video_interval == 0:
      print('Creating video at {}k iterations...'.format(int(step/1000)))

      num_episodes = 3
      video_filename = FLAGS.root_dir + '/videos/sac' + FLAGS.env_name[:-3] + str(int(step/1000)) + 'k.mp4'

      with imageio.get_writer(video_filename, fps=60) as video:
        for _ in range(num_episodes):
          time_step = eval_env.reset()
          video.append_data(eval_py_env.render())
          while not time_step.is_last():
            action_step = tf_agent.policy.action(time_step)
            time_step = eval_env.step(action_step.action)
            video.append_data(eval_py_env.render())

      embed_mp4(video_filename)

    if step % plot_interval == 0:
      print("Plotting returns...")
      steps, returns = zip(*returnsCache)
      # steps = [x - 1100000 for x in steps]
      plt.plot(steps, returns)
      plt.ylabel('Average Return')
      plt.xlabel('Step')
      plt.ylim()
      plt.savefig(FLAGS.root_dir + '/plots/sac' + FLAGS.env_name[:-3] + str(int(step/1000)) + 'k.png')
      print("Done plotting...")

if __name__ == '__main__':
  print("Running SAC...")
app.run(main)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
import practice.filechecker
import practice.uploaded_file

class Migration(migrations.Migration):

    dependencies = [
        ('practice', '0006_auto_20150428_1648'),
    ]

    operations = [
        migrations.AlterField(
            model_name='documents',
            name='inter_cert',
            field=practice.filechecker.ContentTypeRestrictedFileField(upload_to=practice.uploaded_file.upload_to_path_ud_inter, verbose_name=b'Inter Certificate'),
            preserve_default=True,
        ),
        migrations.AlterField(
            model_name='documents',
            name='ssc_cert',
            field=practice.filechecker.ContentTypeRestrictedFileField(upload_to=practice.uploaded_file.upload_to_path_ud_ssc, verbose_name=b'SSC Certificate'),
            preserve_default=True,
        ),
    ]


################################################################################
# Copyright IBM Corporation 2021, 2022
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################

"""
Process facts Module

This module processes the facts from the code analysis
"""

import errno
import json
import os
import re
from csv import reader
from pathlib import Path
from typing import Dict, Tuple

import pandas as pd

from dgi.utils.parse_config import Config

class ConsumeFacts:
    "Synthesize DOOP facts"

    def __init__(self, conf: Config) -> None:
        """Take raw datalog facts and convert them to a consumable form.

        Args:
            p_root (str): Path to the project root folder.
            facts_dir (str): Path to the facts folder.
            opt_cfg (Config): Other configs
        """
        self.conf = conf
        self.method_info = {}
        self.__setup()

    def __setup(self) -> None:
        """Perform some setup for auxiliary paths, data structures, etc."""
        self.absolute_facts_dir = Path(self.conf.GRAPH_FACTS_DIR)
        self.method_info = {}
        self.contexts = {}

    def _jsonify_method_string(self, raw: str) -> str:
        """Convert doop style method information to a json formatted string

        Args:
            raw (str): Method information as a doop style string

        Returns:
            str: JSON string representation of the method information
        """

        # In the method info already exists, use that...
        raw = re.sub("[(:)]", " ", raw[1:-1])
        try:
            class_name, return_type, method_sig, _ = (*raw.split(),)
        except ValueError:
            class_name, return_type, method_sig = (*raw.split(),)

        key = "::".join([class_name, method_sig])
        if key in self.method_info:
            method_dict = self.method_info[key]
        else:
            # If not, add a new instance to the dictionary
            method_dict = {
                "name": method_sig,
                "class": class_name,
                "return_type": return_type,
            }
            self.method_info[key] = method_dict

        return method_dict

    def _jsonify_context(self, raw: str) -> str:
        """Convert context string to a json string.

        Args:
            ctx_list (str): Doop format context string

        Returns:
            str: JSON string representation of the doop context

        Notes:
            Take the context information and convert it to:
            "{
                {
                    "class": "class_name_1",
                    "method": "method_name_1",
                    "object": "obj_name_1",
                    "instance": id1,
                },
                {
                    "class": "class_name_2",
                    "method": "method_name_2",
                    "object": "obj_name_2",
                    "instance": id2,
                }
            }"
        """

        raw_str = re.sub(r"[\[\]]", "", raw)  # noqa: W601
        raw_ctx_lst = raw_str.split(", ")

        for i, str_el in enumerate(raw_ctx_lst):
            if str_el in (
                "<<immutable-context>>",
                "<<immutable-hcontext>>",
                "<<string-builder>>",
                "<<string-buffer>>",
            ):
                raw_ctx_lst[i] = {
                    "class": None,
                    "method": str_el,
                    "type": None,
                    "object": None,
                    "instance": 0,
                }
            elif "MockObject" in str_el:
                class_name, object_name = (*str_el.split("::"),)
                method_name = None
                raw_ctx_lst[i] = {
                    "class": class_name,
                    "method": method_name,
                    "type": None,
                    "object": object_name,
                    "instance": 0,
                }
            else:
                raw_substr = raw_ctx_lst[i].split("/")
                class_name, method_signature = (
                    *raw_substr[0][1:-1].split(": "),)
                try:
                    method_rtype, method_name, _ = (
                        *re.sub("[(:)]", " ", method_signature).split(),
                    )
                except ValueError:
                    method_rtype, method_name = (
                        *re.sub("[(:)]", " ", method_signature).split(),
                    )

                object_name = raw_substr[1]
                instance_id = raw_substr[2]
                raw_ctx_lst[i] = {
                    "class": class_name,
                    "method": method_name,
                    "type": method_rtype,
                    "object": object_name,
                    "instance": int(instance_id),
                }

        ctx_json_str = json.dumps(raw_ctx_lst)

        # Add the found context to a dictionary.
        # NOTE: For now, we keep the values empty. In a later method (`_update_context_transistions(...)`),
        # we update it lazily.
        self.contexts.update({ctx_json_str: {"prev": [], "next": []}})

        return raw_ctx_lst

    @staticmethod
    def _jsonify_heap_obj(heapobj_str: str) -> str:
        """Create a JSON string from raw heap object string from doop

        Args:
            heapobj_str (str): Heap object as a string

        Returns:
            str: JSON string
        """
        raw_substr = re.sub("[<:>]", "", heapobj_str)
        if raw_substr in {
            "string-constant",
            "string-builder",
            "java.lang.StringMockObject",
            "null pseudo heap",
        }:
            heap_obj_dict = {
                "class": raw_substr,
                "method": None,
                "object": None,
                "instance": 0,
            }
        elif "MockObject" in heapobj_str:
            class_name, object_name = (*heapobj_str.split("::"),)
            method_name = None
            heap_obj_dict = {
                "class": class_name,
                "method": method_name,
                "object": object_name,
                "instance": 0,
            }
        else:
            raw_substr = raw_substr.split("/")
            class_name, _, method_signature = (*raw_substr[0].split(" "),)

            try:
                method_rtype, method_name = (*method_signature.split(),)
            except ValueError:
                method_name = method_signature.split()[0]
                method_rtype = (None,)
            try:
                object_name = raw_substr[1].split()[1]
                instance_id = raw_substr[2]
            except IndexError:
                object_name = raw_substr[0].split()[0]
                instance_id = None
            heap_obj_dict = {
                "class": class_name,
                "method": method_name,
                "type": method_rtype,
                "object": object_name,
                "instance": int(instance_id),
            }

        return heap_obj_dict

    def _process_method_info(self, method_info_file: Path) -> None:
        """Process method information into a dictionary for reference

        Args:
            method_info_file (Path): Path to doop method information file
        """
        with open(method_info_file, "r", encoding='utf-8') as file_obj:
            csv_reader = reader(file_obj, delimiter="\t")
            for row in csv_reader:
                key = "::".join(row[::-1][1:3])
                if key not in self.method_info:
                    self.method_info[key] = {
                        "name": row[1],
                        "class": row[2],
                        "return_type": row[3],
                    }

    def _process_heap_carried_dependencies(self, fact_loc: Path) -> pd.DataFrame:
        """Process heap carried dependencies

        Args:
            fact_loc (Path): Path to the heap carried dependency flows file.

        Returns:
            pd.DataFrame: The dataframe containing processed heap carried dependencies
        """
        heap_flows_df = pd.read_csv(fact_loc, header=None, delimiter="\t")
        heap_flows_df.columns = ["context", "heap_obj", "prev", "next"]
        heap_flows_df.context = heap_flows_df.context.apply(
            self._jsonify_context)
        heap_flows_df.prev = heap_flows_df.prev.apply(
            self._jsonify_method_string)
        heap_flows_df.next = heap_flows_df.next.apply(
            self._jsonify_method_string)
        heap_flows_df.heap_obj = heap_flows_df.heap_obj.apply(
            self._jsonify_heap_obj)
        return heap_flows_df

    def _process_call_return_dependencies(
        self, calls_fact_loc: Path, returns_fact_loc: Path
    ) -> pd.DataFrame:
        """Process call-return dependencies

        Args:
            calls_fact_loc (Path): Path to the call dependencies file.
            returns_fact_loc (Path): Path to the return dependencies file.

        Returns:
            pd.DataFrame: The dataframe containing processed call-return dependencies
        """
        callret_flows_df = pd.read_csv(
            calls_fact_loc, header=None, delimiter="\t")
        pd.concat(
            [
                callret_flows_df,
                pd.read_csv(returns_fact_loc, header=None, delimiter="\t"),
            ]
        )
        callret_flows_df.columns = [
            "prev_context", "prev", "next_context", "next"]
        callret_flows_df.prev_context = callret_flows_df.prev_context.apply(
            self._jsonify_context
        )
        callret_flows_df.next_context = callret_flows_df.next_context.apply(
            self._jsonify_context
        )
        callret_flows_df.prev = callret_flows_df.prev.apply(
            self._jsonify_method_string)
        callret_flows_df.next = callret_flows_df.next.apply(
            self._jsonify_method_string)
        return callret_flows_df

    def _process_data_dependencies(self, fact_loc: Path) -> pd.DataFrame:
        """Process data dependent instructions

        Args:
            fact_loc (Path): Path to the data dependent flows.

        Returns:
            pd.DataFrame: The dataframe containing processed data dependencies
        """
        data_flows_df = pd.read_csv(fact_loc, header=None, delimiter="\t")
        data_flows_df.columns = ["context", "prev", "next"]
        data_flows_df.context = data_flows_df.context.apply(
            self._jsonify_context)
        data_flows_df.prev = data_flows_df.prev.apply(
            self._jsonify_method_string)
        data_flows_df.next = data_flows_df.next.apply(
            self._jsonify_method_string)
        return data_flows_df

    def get_method_info(self) -> Dict:
        """Method information as a dictionary

        Returns:
            Dict: Method information
        """
        if not self.method_info:
            method_info_file = self.absolute_facts_dir.joinpath(
                self.conf.METHOD_INFORMATION_FILE
            )
            self._process_method_info(method_info_file)

        return self.method_info

    def get_contexts(self) -> set:
        """Get context information as a dictionary

        Returns:
            Dict: Context information
        """

        assert (
            self.contexts
        ), "Context information has not yet been derived. Run `process_and_get_facts_data()` first."

        return self.contexts

    def process_and_get_facts_data(
        self,
    ) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
        """Process all facts and return the data

        Returns:
            Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: Heap flows, data flows, and call/return flows.
        """

        # ------------------
        # Method information
        # ------------------
        method_info_file = self.absolute_facts_dir.joinpath(
            self.conf.METHOD_INFORMATION_FILE
        )

        if not method_info_file.exists():
            raise FileNotFoundError(
                errno.ENOENT, os.strerror(errno.ENOENT), str(method_info_file)
            )

        self._process_method_info(method_info_file)

        # -----------------------------
        # Heap carried dependency flows
        # -----------------------------
        heap_facts_file = self.absolute_facts_dir.joinpath(
            self.conf.HEAP_DEPENDENCY_FILE
        )

        if not heap_facts_file.exists():
            raise FileNotFoundError(
                errno.ENOENT, os.strerror(errno.ENOENT), str(heap_facts_file)
            )

        heap_flows = self._process_heap_carried_dependencies(heap_facts_file)

        # ---------------------
        # Data dependency flows
        # ---------------------
        data_dep_facts_file = self.absolute_facts_dir.joinpath(
            self.conf.DATA_DEPENDENCY_FILE
        )

        if not data_dep_facts_file.exists():
            raise FileNotFoundError(
                errno.ENOENT, os.strerror(
                    errno.ENOENT), str(data_dep_facts_file)
            )

        data_flows = self._process_data_dependencies(data_dep_facts_file)

        # ----------------------------
        # Call-return dependency flows
        # ----------------------------
        call_dep_facts_file = self.absolute_facts_dir.joinpath(
            self.conf.CALL_DEPENDENCY_FILE
        )
        return_dep_facts_file = self.absolute_facts_dir.joinpath(
            self.conf.RETURN_DEPENDENCY_FILE
        )

        if not call_dep_facts_file.exists():
            raise FileNotFoundError(
                errno.ENOENT, os.strerror(
                    errno.ENOENT), str(call_dep_facts_file)
            )

        if not return_dep_facts_file.exists():
            raise FileNotFoundError(
                errno.ENOENT, os.strerror(
                    errno.ENOENT), str(return_dep_facts_file)
            )

        call_return_flows = self._process_call_return_dependencies(
            call_dep_facts_file, return_dep_facts_file
        )

        return heap_flows, data_flows, call_return_flows


GIFTS = {1: 'a Partridge in a Pear Tree', 2: 'two Turtle Doves', 3: 'three French Hens',
        4: 'four Calling Birds', 5: 'five Gold Rings', 6: 'six Geese-a-Laying',
        7: 'seven Swans-a-Swimming', 8: 'eight Maids-a-Milking', 9: 'nine Ladies Dancing',
        10: 'ten Lords-a-Leaping', 11: 'eleven Pipers Piping', 12: 'twelve Drummers Drumming'}

DAYS = ['first', 'second', 'third', 'fourth', 'fifth', 'sixth',
        'seventh', 'eighth', 'ninth', 'tenth', 'eleventh', 'twelfth']

def recite(start_verse, end_verse):
    lyrics = []
    for day in range(start_verse, end_verse + 1):
        verse = f'On the {DAYS[day-1]} day of Christmas my true love gave to me: '
        for gift in range(day, 0, -1):
            verse = verse + ('and ' if (gift == 1 and day > 1) else '')
            verse = verse + GIFTS[gift]
            verse = verse + ('.' if gift == 1 else ', ')
        lyrics.append(verse)
    return lyrics


from django.conf.urls.defaults import *
from django.conf import settings
from django.contrib import admin
from django.contrib.auth.views import login, logout, logout_then_login, password_change
from django.contrib.auth.forms import AuthenticationForm
from haystack.views import SearchView
from wehaveweneed.search.forms import PostSearchForm
from wehaveweneed.web.forms import RegistrationForm

admin.autodiscover()

urlpatterns = patterns('',
    url(r'^accounts/activate/(?P<activation_key>\w+)/$',
        'registration.views.activate',
        {'extra_context': {'auth_form': AuthenticationForm()}},
        name='registration_activate'),
    #url(r'^accounts/register/$', 'registration.views.register', {'form_class': RegistrationForm}),
    url(r'^accounts/settings/$', 'wehaveweneed.web.views.account_settings'),
    url(r'^accounts/', include('registration.urls')),
    url(r'^admin/', include(admin.site.urls)),
    url(r'^api/', include('wehaveweneed.api.urls')),
    url(r'^feeds/', include('wehaveweneed.api.feedurls')),
    url(r'^login/', login, { 'template_name': 'registration/login.html' }),
    url(r'^logout/', logout_then_login ),
    url(r'^haves/(?P<category>[-\w]+)?', 'wehaveweneed.web.views.viewhaves', name='web_viewhaves'),
    url(r'^needs/(?P<category>[-\w]+)?', 'wehaveweneed.web.views.viewneeds', name='web_viewneeds'),
    url(r'^post/', 'wehaveweneed.web.views.post_create', name='web_postcreate'),
    url(r'^search/', SearchView(form_class=PostSearchForm)),
    url(r'^top_needs/$', 'wehaveweneed.web.views.top_needs', name='top_needs'),
    url(r'^view/(?P<id>\d+)/$', 'wehaveweneed.web.views.view_post', name='view_post'),
    url(r'^$', 'wehaveweneed.web.views.home', name="home"),
)

if (settings.DEBUG):
    urlpatterns += patterns('',
        url(r'^%s/(?P<path>.*)$' % settings.MEDIA_URL.strip('/'), 'django.views.static.serve', {'document_root': settings.MEDIA_ROOT}),
    )


from __future__ import print_function

filename = ("teste.txt")

class Memoria:
    def __init__(self, buscaInstrucao, decodificacao, execucao, escritaResultado, clock):
        self.buscaInstrucao = buscaInstrucao
        self.decodificacao = decodificacao
        self.execucao = execucao
        self.escritaResultado = escritaResultado
        self.clock = 0
    def mostraSituacao(self):
        print("Clock = %s\nBusca de instrucao = %s\nDecodificacao = %s\nExecucao = %s\nEscrita dos resultados = %s"%
        (self.clock, self.buscaInstrucao, self.decodificacao, self.execucao, self.escritaResultado))

m = Memoria("","","","", 0)
registrador = []

for i in range(1,34):
    registrador.append(0)

nline = 100
with open(filename) as f:
    for i, line in enumerate(f, start=1):
        if i > nline:
            break

        line = line.split()
        m.escritaResultado = "".join(m.execucao)
        m.execucao = "".join(m.decodificacao)
        m.decodificacao = "".join(m.buscaInstrucao)
        m.buscaInstrucao = " ".join(line)
        m.clock += 1
        m.mostraSituacao()

        if len(line) > 0:
            if line[0]=="move":
                a = int(line[1][2])
                b = int(line[2][2])
                registrador[a] = registrador[b]

            elif line[0]=="add":
                a = int(line[1][2])
                b = int(line[2][2])
                c = int(line[3][2])
                registrador[a] = registrador[b] + registrador[c]
                m.clock += 1

            elif line[0]=="addi":
                a = int(line[1][2])
                b = int(line[2][2])
                c = int(line[3])
                registrador[a] = registrador[b] + c

            elif line[0]=="sub":
                a = int(line[1][2])
                b = int(line[2][2])
                c = int(line[3][2])
                registrador[a] = registrador[b] - registrador[c]
                m.clock += 1
            else:
                a = int(line[1][2])
                b = int(line[2][2])
                c = int(line[3])
                registrador[a] = registrador[b] - c
        for j in range(1,33):
            print("$r%s: %s"%(j, registrador[j]))


from datetime import datetime, timedelta
from timely_beliefs.beliefs.utils import load_time_series
from scipy.special import erfinv
from bokeh.palettes import viridis
from bokeh.io import show
from bokeh.models import ColumnDataSource, FixedTicker, FuncTickFormatter, LinearAxis
from bokeh.plotting import figure
import time, pickle
import datetime
import pytz
import isodate
import pandas as pd
import timely_beliefs as tb
import numpy as np
import scipy.stats as stats

# code to read in the csv file:
def read_beliefs_from_csv(sensor, source, cp, event_resolution: timedelta, tz_hour_difference: float = 0) -> list:
    """
    Returns a timely_beliefs DataFrame read from a csv file

    @param sensor : beliefsensor
    @param source : BeliefSource
    @param cp : float, cummulative probability
    @param event_resolution : timedelta object, event resolution
    @param tz_hour_difference : float,time difference
    """
    sensor_descriptions = (("Temperature", "°C"),)

    cols = [0, 1]  # Columns with datetime index and observed values
    horizons = list(range(0, 169, 1))
    cols.extend([h + 2 for h in horizons])
    n_horizons = 169
    n_events = None
    beliefs = pd.read_csv("%s-%s-%s.csv" % (sensor.name.replace(' ', '_').lower(), source.name.replace(' ', '_').lower(), cp),
                          index_col=0, parse_dates=[0], date_parser=lambda col: pd.to_datetime(col, utc=True) - timedelta(hours=tz_hour_difference),
                          nrows=n_events, usecols=cols)
    beliefs = beliefs.resample(event_resolution).mean()
    assert beliefs.index.tzinfo == pytz.utc
    # Construct the BeliefsDataFrame by looping over the belief horizons
    blfs = load_time_series(beliefs.iloc[:, 0].head(n_events), sensor=sensor, source=source,
                            belief_horizon=timedelta(hours=0), cumulative_probability=0.5)  # load the observations (keep cp=0.5)
    for h in beliefs.iloc[:, 1 :n_horizons + 1] :
        try:
            blfs += load_time_series(beliefs[h].head(n_events), sensor=sensor, source=source,
                                     belief_horizon=(isodate.parse_duration(
                                     "PT%s" % h)) + event_resolution, cumulative_probability=cp)  # load the forecasts
        except isodate.isoerror.ISO8601Error:  # In case of old headers that don't yet follow the ISO 8601 standard
            blfs += load_time_series(beliefs[h].head(n_events), sensor=sensor, source=source,
                                     belief_horizon=(isodate.parse_duration(
                                     "%s" % h)) + event_resolution, cumulative_probability=cp)  # load the forecasts
    return blfs

def make_df(n_events = 100, n_horizons = 169, tz_hour_difference=-9, event_resolution=timedelta(hours=1)):
    """
    Returns DataFrame in which n events and n horizons are stored

    @param n_events: int,number of events in DataFrame
    @param n_horizons: int,number of horizons in DataFrame
    @param tz_hour_difference: float,time difference
    @param event_resolution: timedelta object,event resolution
    """
    sensor_descriptions = (("Temperature", "°C"),)
    source = tb.BeliefSource(name="Random forest")
    sensors = (tb.Sensor(name=descr[0], unit=descr[1], event_resolution=event_resolution) for descr in sensor_descriptions)
    blfs=[]
    for sensor in sensors:
        blfs += read_beliefs_from_csv(sensor, source=source, cp=0.05, event_resolution=event_resolution, tz_hour_difference=tz_hour_difference)
        blfs += read_beliefs_from_csv(sensor, source=source, cp=0.5, event_resolution=event_resolution, tz_hour_difference=tz_hour_difference)
        blfs += read_beliefs_from_csv(sensor, source=source, cp=0.95, event_resolution=event_resolution, tz_hour_difference=tz_hour_difference)
        bdf = tb.BeliefsDataFrame(sensor=sensor, beliefs=blfs).sort_index()
    return bdf
    # end of code to read in csv file

def create_cp_data(df, start, end, start_time, fixedviewpoint):
    """
    Returns 3 lists with values of 2 different cumulative probabilities.
    Solely 1 out of 3 is 0.5.

    @param df: BeliefsDataFrame, containing events, belief times, predictions and their cumulative probabilities of 0.05/0.5/0.95
    @param start: int,start of timedelta in hours
    @param end: int,end of timedelta in hours
    @param start_time: datetime object, start of event
    @param fixedviewpoint: BOOLEAN,if true plot based on future predictions
    """
    first_date = df.iloc[0].name[0]
    last_date = df.iloc[-1].name[0]
    # check if current time is in data frame
    if start_time < first_date or start_time > last_date :
        raise ValueError('Your start time is not in the dataframe')
    # get cp for fixed viewpoint or not
    if fixedviewpoint == True:
        df = df[df.index.get_level_values("event_start") >= start_time]
        bdf = df.fixed_viewpoint(start_time)
        end = len(bdf)
        cp0 = bdf.iloc[0].name[3]
        cp1 = bdf.iloc[1].name[3]
        cp2 = bdf.iloc[2].name[3]
    else:
        bdf = df.belief_history(event_start=start_time, belief_horizon_window=(timedelta(hours=start), timedelta(hours=end)))
        cp0 = bdf.iloc[0].name[2]
        cp1 = bdf.iloc[1].name[2]
        cp2 = bdf.iloc[2].name[2]
    # list of cps
    cp_list = [cp0, cp1, cp2]
    list_0 = []
    list_1 = []
    list_2 = []
    # make list for each cumulative probability; has to include 0.5
    if 0.5 in cp_list:
        i = 0
        index = cp_list.index(0.5)
        for _, value in bdf.iterrows():
            i = i%3
            if i == 0:
                list_0  += [value[0]]
            elif i == 1:
                list_1 += [value[0]]
            elif i == 2:
                list_2 += [value[0]]
            i += 1
        cp_list.remove(0.5)
        lists = [list_0, list_1, list_2]
        mean_list = lists.pop(index)
        return (cp_list, mean_list, lists[0], lists[1])
    raise ValueError("No mean cp value")

def ridgeline_plot(start_time, df, start=0, end=168, fixedviewpoint = False):
    """
    Creates ridgeline plot by selecting a belief history about a specific event

    @param start_time : datetime string of selected event
    @param df : timely_beliefs DataFrame
    @param start : start of hours before event time
    @param end : end of hours before event time
    @param fixedviewpoint : if true create fixed viewpoint plot
    """
    # set parameters
    if fixedviewpoint == True:
        start = 0
    # out of bounds checks
    if end < 0 or end > 168:
        raise ValueError("End of the forecast horizon must be between 0 and 168 hours.")
    if start < 0 or start > end:
        raise ValueError("Start of the forecast horizon must be between 0 and 168 hours.")
    #to include last observation
    end += 1
    #get cps
    cp_list, pred_temp_05, pred_temp_0, pred_temp_2 = create_cp_data(df,start,end,start_time,fixedviewpoint)
    # make means and standard deviation
    mean = np.array([float(i) for i in pred_temp_05])
    std1 = np.array([(float(pred_temp_0[i])-float(pred_temp_05[i]))/(np.sqrt(2)*erfinv((2*cp_list[0])-1)) for i in range(len(pred_temp_05))])
    std2 = np.array([(float(pred_temp_2[i])-float(pred_temp_05[i]))/(np.sqrt(2)*erfinv((2*cp_list[1])-1)) for i in range(len(pred_temp_05))])
    std = (std1+std2)/2
    #plot everything
    show_plot(mean, std, start, end, fixedviewpoint)

def show_plot(mean, std, start, end, fixedviewpoint=False):
    """
    Creates and shows ridgeline plot
    @param mean: list of float, mean values
    @param std: list of float, standard deviation values
    @param start: int,start hours before event-time
    @param end: int,end hours before event-time
    @param fixedviewpoint : BOOLEAN, if true create fixed viewpoint plot
    """
    # amount of lines to draw
    nr_lines = len(mean)
    x = np.linspace(-10, 30, 500)
    frame = pd.DataFrame()
    # generate points for each line
    for i in range(nr_lines):
        frame["{}".format(i)] = stats.norm.pdf(x, mean[i], std[i])
    #set color pallete to viridis
    pallete = viridis(nr_lines)
    # set list reversed or not depending on viewpoint
    if fixedviewpoint:
        cats = list(frame.keys())
    else:
        cats = list(reversed(frame.keys()))
    source = ColumnDataSource(data=dict(x=x))
    # create figure
    p = figure(y_range=cats, plot_width=900, x_range=(-5, 30), toolbar_location=None)
    # create a ridgeline in the figure
    for i, cat in enumerate(reversed(cats)):
        y = ridge(cat, frame[cat], 50)
        source.add(y, cat)
        p.patch('x', cat, alpha=0.6, color=pallete[i], line_color="black", source=source)
    # added y axis with the right ticks and lables depending on fixedviewpoint
    if fixedviewpoint:
        p.yaxis.axis_label = 'Upcoming hours'
        y_ticks = list(np.arange(0, nr_lines, 6))
        yaxis = LinearAxis(ticker=y_ticks)
        y_labels = list((np.arange(0, nr_lines, 6)))
    else:
        p.yaxis.axis_label = 'Previous hours'
        y_ticks = list(np.arange(0, end, 6))[::-1]
        yaxis = LinearAxis(ticker=y_ticks)
        y_labels = list(np.arange(start, end, 6))
    # map ticks on to a dict
    mapping_dict = {y_ticks[i]: str(y_labels[i]) for i in range(len(y_labels))}
    for i in range(nr_lines):
        if i not in mapping_dict:
            mapping_dict[i]=" "
    mapping_code = "var mapping = {};\n    return mapping[tick];\n    ".format(mapping_dict)
    #format ticks
    p.yaxis.formatter = FuncTickFormatter(code=mapping_code)
    #set plot atributes
    p.outline_line_color = None
    p.background_fill_color = "#ffffff"
    p.xaxis.ticker = FixedTicker(ticks=list(range(-20, 101, 10)))
    p.xaxis.axis_label = 'Temperature (°C)'
    p.ygrid.grid_line_color = None
    p.xgrid.grid_line_color = "#000000"
    p.xgrid.ticker = p.xaxis[0].ticker
    p.axis.minor_tick_line_color = None
    p.axis.major_tick_line_color = None
    p.axis.axis_line_color = None
    #add padding in y derection
    p.y_range.range_padding = 0.2 / (nr_lines / 168)
    show(p)

# creates a list for the ridgeline function as input
def ridge(category, data, scale=100):
    return list(zip([category] * len(data), scale * data))

# tests
df = make_df()
ridgeline_plot(datetime.datetime(2015, 3, 1, 9, 0, tzinfo=pytz.utc), df, end=150, fixedviewpoint=False)
ridgeline_plot(datetime.datetime(2015, 3, 1, 9, 0, tzinfo=pytz.utc), df, fixedviewpoint=True)


import pytest
from wavedrom import render

@pytest.fixture
def make_test():
    def _make_test(name):
        inputfile = "test/files/{}.json".format(name)

        with open(inputfile, "r") as f:
            jinput = f.read()

        output = render(jinput)

    return _make_test


import pandas as pd
from bs4 import BeautifulSoup
from instagramy import  InstagramUser
import requests
import numpy as np
session_id ="1788a069f20-4afb8e"

def extract(keyword):

    headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36'}
    url = 'https://www.searchmy.bio/search?q='+ keyword

    r=requests.get(url).text
    soup=BeautifulSoup(r,'lxml')
    divs=soup.find_all('div',class_='search-result')
    for items in divs:
        username=items.find('div',class_='search-result-username').text
        fullname=items.find('div',class_='search-result-fullname').text
        followers=items.find('div',class_='search-result-data-point').text
        profile_link=(items.find('a',class_='search-result-button')['href'])
        biography=items.find('div',class_='search-result-bio').text
        userdetails={
        'username':username,
        'fullname':fullname,
        'followers':followers,
        'profile_link':profile_link,
        'biography':biography

              }
        userdata.append(userdetails)
    divs_more=soup.find_all('read-more',class_='ng-scope ng-isolate-scope')
    for items in divs_more:
        user_name = items.find('div', class_='search-result-username ng-binding').text
        full_name = items.find('div', class_='search-result-fullname ng-binding').text
        total_followers = items.find('div', class_='search-result-data-point ng-binding').text
        profile_link_instagram = (items.find('a', class_='search-result-button')['href'])
        biography_profile = items.find('div', class_='search-result-bio ng-binding').text
        user_details = {
            'username': user_name,
            'fullname': full_name,
            'followers': total_followers,
            'profile_link': profile_link_instagram,
            'biography': biography_profile

        }
        userdata.append(user_details)

    return

userdata=[]

extract("design")

df=pd.DataFrame(userdata)
print(df.head())
df.to_csv('userdata.csv')


# coding:utf-8
#
# The MIT License (MIT)
#
# Copyright (c) 2016-2018 yutiansut/QUANTAXIS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import random
import sched
import threading
import time

import pandas as pd

from QUANTAXIS.QAEngine.QAEvent import QA_Event, QA_Worker
from QUANTAXIS.QAEngine.QATask import QA_Task
from QUANTAXIS.QAMarket.QAOrder import QA_OrderQueue
from QUANTAXIS.QASU.save_orderhandler import QA_SU_save_deal, QA_SU_save_order
from QUANTAXIS.QAUtil.QADate_trade import QA_util_if_tradetime
from QUANTAXIS.QAUtil.QAParameter import (BROKER_EVENT, BROKER_TYPE,
                                          EVENT_TYPE, MARKET_EVENT,
                                          ORDER_EVENT)

class QA_OrderHandler(QA_Worker):
    """ORDER执行器

    ORDEHANDLDER 归属于MARKET前置

    仅负责一个无状态的执行层

    ORDER执行器的作用是因为
    在实盘中 当一个订单发送出去的时候,市场不会返回一个更新的订单类回来
    大部分时间都依赖子线程主动查询 或者是一个市场信息来进行判断

    ORDER_Handler的作用就是根据信息更新Order

    用于接受订单 发送给相应的marker_broker 再根据返回的信息 进行更新

    可用的market_broker:
    1.回测盘
    2.实时模拟盘
    3.实盘

    ORDERHANDLER 持久化问题:

    设定机制: 2秒查询1次
    持久化: 2秒一次

    2018-07-29

    # 重新设置ORDERHADLER的运行模式:

    -- 常规检查 5秒一次

    -- 如果出现订单 则2-3秒 对账户轮询(直到出现订单成交/撤单为止)

    """

    def __init__(self, *args, **kwargs):
        super().__init__()
        self.order_queue = QA_OrderQueue()
        self.type = EVENT_TYPE.MARKET_EVENT

        self.event = QA_Event()
        self.order_status = pd.DataFrame()
        self.deal_status = pd.DataFrame()
        self.if_start_orderquery = False

    def run(self, event):
        if event.event_type is BROKER_EVENT.RECEIVE_ORDER:
            # 此时的message应该是订单类
            """
            OrderHandler 收到订单

            orderhandler 调控转发给broker

            broker返回发单结果(成功/失败)

            orderhandler.order_queue 插入一个订单

            执行回调

            """

            order = event.order
            # print(event.broker)
            order = event.broker.receive_order(
                QA_Event(event_type=BROKER_EVENT.TRADE, order=event.order))
            # print(threading.current_thread().ident)
            order = self.order_queue.insert_order(order)
            if event.callback:
                event.callback(order)

        elif event.event_type is BROKER_EVENT.TRADE:
            # 实盘和本地 同步执行
            res = []
            for order in self.order_queue.pending:
                result = event.broker.query_orders(
                    order.account_cookie, order.realorder_id)
                self.order_queue.set_status(
                    order.order_id, result['header']['status'])
                if order.callback:
                    order.callback(result)
                res.append(result)
            event.res = res

            return event

        elif event.event_type is BROKER_EVENT.SETTLE:
            self.order_queue.settle()

        elif event.event_type is MARKET_EVENT.QUERY_ORDER:
            """query_order和query_deal 需要联动使用

            query_order 得到所有的订单列表

            query_deal 判断订单状态--> 运行callback函数

            实盘涉及到外部订单问题:
            及 订单的来源 不完全从QUANTAXIS中发出, 则QA无法记录来源 (标记为外部订单)
            """

            if self.if_start_orderquery:
                # if QA_util_if_tradetime(datetime.datetime.now()):

                    # print(event.broker)
                    # print(event.account_cookie)
                try:
                    # 做一些容错处理
                    res = [event.broker[i].query_orders(
                        event.account_cookie[i], '') for i in range(len(event.account_cookie))]
                    res = pd.concat(res, axis=0) if len(
                        res) > 0 else None

                except:
                    time.sleep(1)

                self.order_status = res if res is not None else self.order_status
                if len(self.order_status) > 0:
                    QA_SU_save_order(self.order_status)
                # else:
                #     time.sleep(1)

            # 这里加入随机的睡眠时间 以免被发现固定的刷新请求
            # event=event
            event.event_type = MARKET_EVENT.QUERY_DEAL
            if event.event_queue.qsize() < 1:
                time.sleep(random.randint(1, 2))

            # 非阻塞
            event.event_queue.put(
                QA_Task(
                    worker=self,
                    engine='ORDER',
                    event=event
                )
            )
            # time.sleep(random.randint(2,5))
            # print(event.event_type)
            # print(event2.event_type)
            # self.run(event)

            # print(self.order_status)
            # print('UPDATE ORDERS')

        elif event.event_type is MARKET_EVENT.QUERY_DEAL:

            """order_handler- query_deal

            将order_handler订单队列中的订单---和deal中匹配起来

            """

            # if len(self.order_queue.pending) > 0:
            #     for order in self.order_queue.pending:
            #         #self.query
            #         waiting_realorder_id = [
            #             order.realorder_id for order in self.order_queue.trade_list]
            #         result = event.broker.query_deal
            #         time.sleep(1)
            if self.if_start_orderquery:
                self.deal_status = [event.broker[i].query_orders(
                    event.account_cookie[i], 'filled') for i in range(len(event.account_cookie))]
                # print(self.order_status)
                self.deal_status = pd.concat(self.deal_status, axis=0) if len(
                    self.deal_status) > 0 else pd.DataFrame()
                if len(self.deal_status) > 0:
                    QA_SU_save_deal(self.deal_status)
                # print(self.order_status)

            # 这里加入随机的睡眠时间 以免被发现固定的刷新请求
            if event.event_queue.qsize() < 1:
                time.sleep(random.randint(2, 5))
            event.event_type = MARKET_EVENT.QUERY_ORDER

            event.event_queue.put(
                QA_Task(
                    worker=self,
                    engine='ORDER',
                    event=event
                )
            )
            # self.run(event)
            # self.run(event)

        elif event.event_type is MARKET_EVENT.QUERY_POSITION:
            pass

    def query_order(self, order_id):
        pass


# Copyright OpenSearch Contributors.
# SPDX-License-Identifier: Apache-2.0

import os
import subprocess
import tempfile
import shutil

class GitRepository:
    '''
    This class checks out a Git repository at a particular ref into a named directory (or temporary a directory if no named directory is given). Temporary directories will be automatically deleted when the GitRepository object goes out of scope; named directories will be deleted and recreated when the GitRepository is constructed.
    Clients can obtain the actual commit ID by querying the "sha" attribute, and the temp directory name with "dir".
    '''
    def __init__(self, url, ref, directory = None):
        self.url = url
        self.ref = ref
        if directory is None:
            self.temp_dir = tempfile.TemporaryDirectory()
            self.dir = self.temp_dir.name
        else:
            self.dir = directory
            print(f'Deleting {self.dir}')
            shutil.rmtree(self.dir, ignore_errors = True)
            os.makedirs(self.dir, exist_ok = True)

        # Check out the repository
        self.execute(f'git init', True)
        self.execute(f'git remote add origin {self.url}', True)
        self.execute(f'git fetch --depth 1 origin {self.ref}', True)
        self.execute(f'git checkout FETCH_HEAD', True)
        self.sha = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd = self.dir).decode().strip()
        print(f'Checked out {self.url}@{self.ref} into {self.dir} at {self.sha}')

    def execute(self, command, silent = False):
        print(f'Executing "{command}" in {self.dir}')
        if silent:
            subprocess.check_call(command, cwd = self.dir, shell = True, stdout = subprocess.DEVNULL, stderr = subprocess.DEVNULL)
        else:
            subprocess.check_call(command, cwd = self.dir, shell = True)