""" run decoding of X-ext (+ abs)"""
import argparse
import json
import os
from os.path import join
from datetime import timedelta
from time import time
from collections import Counter, defaultdict
from itertools import product
from functools import reduce
import operator as op

from cytoolz import identity, concat, curry

import torch
from torch.utils.data import DataLoader
from torch import multiprocessing as mp

from data.batcher import tokenize

from decoding import Abstractor, Extractor, DecodeDataset, BeamAbstractor, ConditionalAbstractor, BeamConditionalAbstractor
from decoding import make_html_safe

MAX_ABS_NUM = 6  # need to set max sentences to extract for non-RL extractor

def decode(save_path, abs_dir, ext_dir, split, batch_size, max_len, beam_size, diverse, cuda, disable_final_rerank=False, is_conditional_abs=False):
    start = time()
    # setup model
    if abs_dir is None:
        # NOTE: if no abstractor is provided then
        #       the whole model would be extractive summarization
        abstractor = identity
    elif is_conditional_abs:
        if beam_size == 1:
            abstractor = ConditionalAbstractor(abs_dir, max_len, cuda)
        else:
            abstractor = BeamConditionalAbstractor(abs_dir, max_len, cuda)
    else:
        #abstractor = Abstractor(abs_dir, max_len, cuda)
        if beam_size == 1:
            abstractor = Abstractor(abs_dir, max_len, cuda)
        else:
            abstractor = BeamAbstractor(abs_dir, max_len, cuda)

    if ext_dir is None:
        # NOTE: if no abstractor is provided then
        #       it would be  the lead-N extractor
        extractor = lambda art_sents: list(range(len(art_sents)))[:MAX_ABS_NUM]
    else:
        extractor = Extractor(ext_dir, max_ext=MAX_ABS_NUM, cuda=cuda)

    # setup loader
    def coll(batch):
        articles = list(filter(bool, batch))
        return articles
    dataset = DecodeDataset(split)

    n_data = len(dataset)
    loader = DataLoader(
        dataset, batch_size=batch_size, shuffle=False, num_workers=4,
        collate_fn=coll
    )

    # prepare save paths and logs
    for i in range(MAX_ABS_NUM):
        os.makedirs(join(save_path, 'output_{}'.format(i)))
    dec_log = {}
    dec_log['abstractor'] = (None if abs_dir is None
                             else json.load(open(join(abs_dir, 'meta.json'))))
    dec_log['extractor'] = (None if ext_dir is None
                            else json.load(open(join(ext_dir, 'meta.json'))))
    dec_log['rl'] = False
    dec_log['split'] = split
    dec_log['beam'] = beam_size
    with open(join(save_path, 'log.json'), 'w') as f:
        json.dump(dec_log, f, indent=4)

    # Decoding
    i = 0
    with torch.no_grad():
        for i_debug, raw_article_batch in enumerate(loader):
            tokenized_article_batch = map(tokenize(None), raw_article_batch)
            num_articles = len(raw_article_batch)
            num_ext_sents = 0
            if is_conditional_abs:
                sequential_ext_sents = []
                sequential_article_ids = []
            else:
                ext_arts = []
            ext_inds = []
            for article_i, raw_art_sents in enumerate(tokenized_article_batch):
                ext = extractor(raw_art_sents)
                ext_inds += [(num_ext_sents, len(ext))]
                num_ext_sents += len(ext)

                if is_conditional_abs:
                    # insert place holder to sequential_ext_sents
                    num_selected_sents_excluded_eos = len(ext)
                    if num_selected_sents_excluded_eos > len(sequential_ext_sents):
                        [sequential_ext_sents.append([]) for _ in
                         range(num_selected_sents_excluded_eos - len(sequential_ext_sents))]
                        [sequential_article_ids.append([]) for _ in
                         range(num_selected_sents_excluded_eos - len(sequential_article_ids))]

                    for idx_i, idx in enumerate(ext):
                        sequential_ext_sents[idx_i].append(raw_art_sents[idx])
                        sequential_article_ids[idx_i].append(article_i)
                else:
                    ext_arts += list(map(lambda i: raw_art_sents[i], ext))

            if beam_size > 1:
                if is_conditional_abs:
                    dec_outs = abstractor(sequential_ext_sents, sequential_article_ids, num_articles, beam_size,
                                          diverse)
                else:
                    all_beams = abstractor(ext_arts, beam_size, diverse)  # a list of beam for the whole batch
                    dec_outs = rerank_mp(all_beams, ext_inds, disable_final_rerank)
            else:
                if is_conditional_abs:
                    dec_outs = abstractor(sequential_ext_sents, sequential_article_ids, num_articles)
                else:
                    dec_outs = abstractor(ext_arts)
            assert i == batch_size*i_debug
            for j, n in ext_inds:
                decoded_sents = [' '.join(dec) for dec in dec_outs[j:j+n]]
                for k, dec_str in enumerate(decoded_sents):
                    with open(join(save_path, 'output_{}/{}.dec'.format(k, i)),
                              'w') as f:
                        f.write(make_html_safe(dec_str))

                i += 1
                print('{}/{} ({:.2f}%) decoded in {} seconds\r'.format(
                    i, n_data, i/n_data*100, timedelta(seconds=int(time()-start))
                ), end='')
    print()

_PRUNE = defaultdict(
    lambda: 2,
    {1:5, 2:5, 3:5, 4:5, 5:5, 6:4, 7:3, 8:3}
)

def rerank(all_beams, ext_inds, disable_final_rerank=False):
    final_rerank = not disable_final_rerank
    beam_lists = (all_beams[i: i+n] for i, n in ext_inds if n > 0)
    if final_rerank:
        return list(concat(map(rerank_one, beam_lists)))
    else:
        return list(concat(map(top_one, beam_lists)))

def rerank_mp(all_beams, ext_inds, disable_final_rerank=False):
    final_rerank = not disable_final_rerank
    beam_lists = [all_beams[i: i+n] for i, n in ext_inds if n > 0]
    # a list of beam list, each beam list contains the beam for one article
    with mp.Pool(8) as pool:
        if final_rerank:
            reranked = pool.map(rerank_one, beam_lists)
        else:
            reranked = pool.map(top_one, beam_lists)
    return list(concat(reranked))

def rerank_one(beams):
    @curry
    def process_beam(beam, n):
        for b in beam[:n]:
            b.gram_cnt = Counter(_make_n_gram(b.sequence))
        return beam[:n]
    beams = map(process_beam(n=_PRUNE[len(beams)]), beams)
    best_hyps = max(product(*beams), key=_compute_score)  # a tuple of
    dec_outs = [h.sequence for h in best_hyps]
    return dec_outs

def top_one(beams):
    return [hyps[0].sequence for hyps in beams]

def _make_n_gram(sequence, n=2):
    return (tuple(sequence[i:i+n]) for i in range(len(sequence)-(n-1)))

def _compute_score(hyps):
    all_cnt = reduce(op.iadd, (h.gram_cnt for h in hyps), Counter())
    repeat = sum(c-1 for g, c in all_cnt.items() if c > 1)
    lp = sum(h.logprob for h in hyps) / sum(len(h.sequence) for h in hyps)
    return (-repeat, lp)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description=('combine an extractor and an abstractor '
                     'to decode summary and evaluate on the '
                     'CNN/Daily Mail dataset')
    )
    parser.add_argument('--path', required=True, help='path to store/eval')
    parser.add_argument('--abs_dir', help='root of the abstractor model')
    parser.add_argument('--ext_dir', help='root of the extractor model')

    # dataset split
    data = parser.add_mutually_exclusive_group(required=True)
    data.add_argument('--val', action='store_true', help='use validation set')
    data.add_argument('--test', action='store_true', help='use test set')

    # decode options
    parser.add_argument('--batch', type=int, action='store', default=32,
                        help='batch size of faster decoding')
    parser.add_argument('--n_ext', type=int, action='store', default=4,
                        help='number of sents to be extracted')
    parser.add_argument('--max_dec_word', type=int, action='store', default=30,
                        help='maximun words to be decoded for the abstractor')

    parser.add_argument('--no-cuda', action='store_true',
                        help='disable GPU training')
    parser.add_argument('--is_conditional_abs', action='store_true', help='use conditional abstractor')

    parser.add_argument('--beam', type=int, action='store', default=1,
                        help='beam size of abstractor decoding (reranking included)')
    parser.add_argument('--div', type=float, action='store', default=1.0,
                        help='diverse ratio for the diverse beam-search')
    parser.add_argument('--disable_final_rerank', action='store_true',
                        help='disable the final rerank of the beam search output by n-gram repeat')

    args = parser.parse_args()
    args.cuda = torch.cuda.is_available() and not args.no_cuda

    data_split = 'test' if args.test else 'val'
    decode(args.path, args.abs_dir, args.ext_dir,
           data_split, args.batch, args.max_dec_word, args.beam, args.div, args.cuda, args.disable_final_rerank, args.is_conditional_abs)


'''
Basic Picture Viewer
====================

This simple image browser demonstrates the scatter widget. You should
see three framed photographs on a background. You can click and drag
the photos around, or multi-touch to drop a red dot to scale and rotate the
photos.

The photos are loaded from the local images directory, while the background
picture is from the data shipped with kivy in kivy/data/images/background.jpg.
The file kivy_multi.kv describes the interface and the file shadow32.png is
the border to make the images look like framed photographs. Finally,
the file android.txt is used to package the application for use with the
Kivy Launcher Android application.

For Android devices, you can copy/paste this directory into
/sdcard/kivy/kivy_multi on your Android device.

The images in the image directory are from the Internet Archive,
`https://archive.org/details/PublicDomainImages`, and are in the public
domain.

'''

import kivy
kivy.require('1.0.6')

from glob import glob
from random import randint
from os.path import join, dirname
from kivy.app import App
from kivy.logger import Logger
from kivy.uix.scatter import Scatter
from kivy.properties import StringProperty
from kivy.uix.popup import Popup
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.button import Button
from kivy.uix.label import Label
from kivy.graphics import *

from kivy.uix.image import Image
from kivy.clock import Clock
from kivy.graphics.texture import Texture
import cv2, time
import numpy as np
from scratch.kivy_multi.videostream import VideoStream, example_VideoCapture
from FLIR_pubsub.FLIR_videocapture import FlirCapture
from FLIR_pubsub.FLIR_client_utils import *

import multiprocessing
from multiprocessing import Process, freeze_support, Lock, current_process, Value, Array
from scratch.kivy_multi.flir_cam_controller \
    import NP_SharedMemory, fake_camera_process, flir_camera_process, SpinProcess
import timeit

class Picture(Scatter):
    '''Picture is the class that will show the image with a white border and a
    shadow. They are nothing here because almost everything is inside the
    picture.kv. Check the rule named <Picture> inside the file, and you'll see
    how the Picture() is really constructed and used.

    The source property will be the filename to show.
    '''

    source = StringProperty(None)

class PicturesApp(App):

    def build(self):
        self.vs1 = VideoStream(src="rtsp://192.168.183.242:554", verbose=True).start()
        # self.vs2 = VideoStream(src="rtsp://192.168.183.242:554", verbose=True).start()
        # self.vs3 = VideoStream(src="rtsp://192.168.183.242:554", verbose=True).start()
        self.vs4 = VideoStream(src="test", name='test', videocapture=example_VideoCapture, verbose=True).start()
        # self.vcap = cv2.VideoCapture("rtsp://192.168.183.242:554")
        self.i = 0

        name = 'FrontLeft'
        url = 'localhost'

        # # shared memory
        self.sm_flir = NP_SharedMemory((2000, 3000, 3), name='camera_1')
        self.sm_flir.set_local_np_view()
        self.sm_fake = NP_SharedMemory((2000, 3000, 3), name='camera_1')
        self.sm_fake.set_local_np_view()
        # sm2 = NP_SharedMemory((2000, 3000, 3), name='camera_2')
        # sm2.set_local_np_view()

        self.proc_fake = SpinProcess(target=fake_camera_process, sharemem=self.sm_fake)
        self.proc_flir = SpinProcess(target=flir_camera_process, sharemem=self.sm_flir)
        # p2 = Process(target=demo_camera_process, args=(sm2,)).start()

        # the root is created in kivy_multi.kv
        root = self.root
        # self.img1 = Image()
        # root.add_widget(self.img1)

        # get any files into images directory
        curdir = dirname(__file__)
        self.stream_widget_list = []
        for filename in glob(join(curdir, 'images', '*')):
            try:
                # load the image
                picture = Picture(source=filename, rotation=randint(-30, 30))
                # add to the main field
                root.add_widget(picture)
                self.stream_widget_list.append(picture)
            except Exception as e:
                Logger.exception('Pictures: Unable to load <%s>' % filename)

        Clock.schedule_interval(self.update_kivy, 1.0 / 30.0)
        # Clock.schedule_interval(self.wastetime, 1.0 / 100.0)

    def wastetime(self, dt):
        # time.sleep(0.01)
        start = time.monotonic()
        while time.monotonic() - start < 0.1:
            pass

    def on_pause(self):
        return True

    def stream2widget(self, vs, _widgit, bg=None):
        _buf = vs.read()
        # time.sleep(1)

        if _buf is None:
            buf = None
            pass
        elif isinstance(_buf, NP_SharedMemory):
            shape = (_buf._height.value, _buf._width.value, 3)
            buf = _buf.arr[:shape[0]*shape[1]*shape[2]]
            buf.shape = shape

        else:
            buf = _buf

        if isinstance(buf, np.ndarray) and buf.size > 100000:
            # if hasattr(buf, 'size') and buf.size > 100000:
            # convert it to texture
            # all ready flattened
            #
            if 1:
                # start = timeit.timeit()
                if len(buf.shape) == 3:
                    texture1 = Texture.create(size=(buf.shape[1], buf.shape[0]), colorfmt='bgr')
                    buf = cv2.flip(buf, 0).flatten()
                    # buf = buf.flatten()
                elif len(buf.shape) == 1:
                    texture1 = Texture.create(size=(3000, 2000), colorfmt='bgr')
                else:
                    raise RuntimeError(f'Received frame shape should be 1 or 3, got: {buf.shape}')

               # end1 = timeit.timeit()
                texture1.blit_buffer(buf, colorfmt='bgr', bufferfmt='ubyte')
                # end2 = timeit.timeit()

                # end2 = timeit.timeit()
                # display image from the texture
                if bg is not None:
                   bg.texture = texture1
                _widgit.children[0].texture = texture1
                # end = timeit.timeit()
                # print('Texture: ', end - start)
            else:
                _widgit.children[0].texture.blit_buffer(buf,
                                                        size=(frame.shape[1], frame.shape[0]),
                                                        colorfmt='bgr', bufferfmt='ubyte')

            # widget.canvas
            # with _widgit.children[0].canvas:  todo jn get this going
            #     Rectangle(texture=_widgit.children[0].texture, pos=_widgit.children[0].pos, size=(64, 64))

    def update_kivy(self, dt):
        '''display image from cam in kivy window'''
        self.stream2widget(self.vs1, self.stream_widget_list[0])
        self.i += 1
        if self.i % 5 == 0:
            # bodge to slow these down
            # self.stream2widget(self.vs2, self.stream_widget_list[1])
            # self.stream2widget(self.vs3, self.stream_widget_list[2])
            # self.stream2widget(self.vs4, self.stream_widget_list[3])
            self.stream2widget(self.proc_fake, self.stream_widget_list[3])
            self.stream2widget(self.proc_flir, self.stream_widget_list[4])
            pass

    def on_request_close(self, *args):
        self.textpopup(title='Exit', text='Are you sure?')
        return True

    def on_stop(self, *args):
        print('!!!! Running on_stop')
        self.proc_flir.release()
        self.proc_fake.release()
        self.vs1.release()
        self.vs4.release()
        return True

    def textpopup(self, title='', text=''):
        """Open the pop-up with the name.

        :param title: title of the pop-up to open
        :type title: str
        :param text: main text of the pop-up to open
        :type text: str
        :rtype: None
        """
        box = BoxLayout(orientation='vertical')
        box.add_widget(Label(text=text))
        mybutton = Button(text='OK', size_hint=(1, 0.25))
        box.add_widget(mybutton)
        popup = Popup(title=title, content=box, size_hint=(None, None), size=(600, 300))
        mybutton.bind(on_release=self.stop)
        popup.open()

import cv2, imutils
import os, sys, logging

if __name__ == '__main__':

    multiprocessing.log_to_stderr()
    logger = multiprocessing.get_logger()
    logger.setLevel(logging.INFO)

    app = PicturesApp().run()

    print('Main: End of program')

    # time.sleep(2)
    print('Main: join')
    # p1.join()
    # p2.join()
    print('Main: bye')
    cv2.destroyAllWindows()


#!/usr/bin/python
from scipy.integrate import odeint
import matplotlib.pyplot as plt # for plotting
import numpy as np
from copy import copy

class Particle (object):

    """Class that describes particle"""
    m = 1.0

    def __init__(self, x0=1.0, v0=0.0,  tf = 10.0, dt = 0.001):
        self.x = x0
        self.v = v0
        self.x0 = x0
        self.v0 = v0
        self.t = 0.0
        self.tf = tf
        self.dt = dt

        self.tlabel = 'time (s)'
        self.xlabel = 'x (m)'
        self.vlabel = 'v (m/s)'

        npoints = int(tf/dt) # always starting at t = 0.0
        self.npoints = npoints
        self.tarray = np.linspace(0.0, tf,npoints, endpoint = True) # include final timepoint
        self.xv0 = np.array([self.x, self.v]) # NumPy array with initial position and velocity

    def F(self, x, v, t):
        # The force on a free particle is 0
        return array([0.0])

    def Euler_step(self):
        """
        Take a single time step using Euler method
        """

        a = self.F(self.x, self.v, self.t) / self.m
        self.x += self.v * self.dt
        self.v += a * self.dt
        self.t += self.dt

    def Verlet_step(self):

        a = self.F(self.x, self.v, self.t)/self.m
        self.x += self.v * self.dt + 0.5 * self.dt**2 * a

        a2 = self.F(self.x, self.v, self.t)/self.m
        self.v += 0.5 * self.dt * (a + a2)

        self.t +=self.dt

    def Euler_trajectory(self):
        """
        Loop over all time steps to construct a trajectory with Euler method
        Will reinitialize euler trajectory everytime this method is called
        """

        x_euler = []
        v_euler = []
        self.x = self.x0
        self.v = self.v0
        self.t = 0.0

        for ii in range(self.npoints):
            v_euler.append(self.v)
            x_euler.append(self.x)
            self.Euler_step()

        self.x_euler = np.array(x_euler)
        self.v_euler = np.array(v_euler)

    def verlet_trajectory(self):
        """
        Loop over all time steps to construct a trajectory with verlet method
        Will reinitialize verlet trajectory everytime this method is called
        """

        x_verlet = []
        v_verlet = []
        self.x = self.x0
        self.v = self.v0
        self.t = 0.0

        for ii in range(self.npoints):
            v_verlet.append(self.v)
            x_verlet.append(self.x)
            self.Verlet_step()

        self.x_verlet = np.array(x_verlet)
        self.v_verlet = np.array(v_verlet)

    def scipy_trajectory(self):
        """calculate trajectory using SciPy ode integrator"""
        self.xv = odeint(self.derivative, self.xv0, self.tarray)

    def derivative(self, xv, t):
        """right hand side of the differential equation"""
        x =xv[0]
        v =xv[1]
        a = self.F(x, v, t) / self.m
        return np.ravel(np.array([v, a]))

    def results(self):
        """
        Print out results in a nice format
        """

        print('Initial: t = 0.0, x = {}, v = {}, dt = {}'.format(self.x0 , self.v0, self.dt))
        print('Current: t = {}, x = {}, v = {}'.format(self.t, self.x , self.v))

        if hasattr(self, 'xv'):
            print('SciPy ODE Integrator:')
            print('t = {} x = {} v = {}'.format(self.tarray[-1], self.xv[-1, 0], self.xv[-1,1]))

    def plot(self, pt = 'trajectory', axs = None):
        """
        Make nice plots of our results
        """
        if not axs:
            fig1 = plt.figure()
            ax1 = fig1.add_subplot(111)

        if hasattr(self,'xv'):

            if pt == 'trajectory':
                if not axs:
                    ax1.plot(self.tarray, self.xv[:, 0], "k", label = 'odeint')
                else:
                    axs.plot(self.tarray, self.xv[:, 0], "k", label = 'odeint')

            if pt == 'phase':
                if not axs:
                    ax1.plot(self.xv[:, 0], self.xv[:, 1], "k",'.', label = 'odeint')
                else:
                    axs.plot(self.xv[:, 0], self.xv[:, 1], "k",'.', label = 'odeint')

        if hasattr(self,'x_euler'):

            if pt == 'trajectory':
                if not axs:
                    ax1.plot(self.tarray, self.x_euler, "b", label = 'euler')
                else:
                    axs.plot(self.tarray, self.x_euler, "b", label = 'euler')

            if pt == 'phase':
                if not axs:
                    ax1.plot(self.x_euler, self.v_euler, "b",'.', label = 'euler')
                else:
                    axs.plot(self.x_euler, self.v_euler, "b",'.', label = 'euler')

        if hasattr(self,'x_verlet'):

            if pt == 'trajectory':
                if not axs:
                    ax1.plot(self.tarray, self.x_verlet, "r", label = 'verlet')
                else:
                    axs.plot(self.tarray, self.x_verlet, "r", label = 'verlet')

            if pt == 'phase':
                if not axs:
                    ax1.plot(self.x_verlet, self.v_verlet, "r",'.', label = 'verlet')
                else:
                    axs.plot(self.x_verlet, self.v_verlet, "r",'.', label = 'verlet')
        if not axs:
            if pt == 'trajectory':
                ax1.set_xlabel(self.tlabel)
                ax1.set_ylabel(self.xlabel)

            if pt == 'phase':
                ax1.set_xlabel(self.xlabel)
                ax1.set_ylabel(self.vlabel)

            ax1.legend()

class Pendulum(Particle):

    """Subclass of Particle Class that describes a pendulum in a harmonic potential"""
    def __init__(self, l = 9.8, m = 1.0, x0 = 0.0 ,v0 = 0.0, tf = 50.0, dt = 0.001):

        super().__init__(x0,v0,tf,dt)
        # for pendulum x = theta [-pi, pi]
        g = 9.8
        omega0 = np.sqrt(g/l)

        self.l = l # length
        self.m = m # mass
        self.omega0 = omega0 # natural frequency

        self.tlabel = 'time ($1/\omega_0$)'
        self.xlabel = '$\\theta$ (radians)'
        self.vlabel = '$\omega$ (radians/s)'

    def Euler_step(self):
        # overload method to wrap x between [-pi,pi]

        Particle.Euler_step(self)
        if self.x > np.pi:
            self.x -= 2*np.pi
        elif self.x < -np.pi:
            self.x += 2*np.pi

    def verlet_step(self):
        # overload method to wrap x between [-pi,pi]

        Particle.verlet_step(self)
        if self.x > np.pi:
            self.x -= 2*np.pi
        elif self.x < -np.pi:
            self.x += 2*np.pi

    # overload method to wrap x between [-pi,pi]
    def scipy_trajectory(self):
        Particle.scipy_trajectory(self)

        x = self.xv[:,0]
        x_new = np.zeros(np.shape(x))
        x_new[0] = x[0]

        # find change in x between each point
        dx = np.diff(x)
        nx = np.shape(x)[0]

        for ii in range(1,nx):
            # reconstruct x array, checking for out of range values
            x_new[ii] = x_new[ii-1]+dx[ii-1]
            if x_new[ii] > np.pi:
                x_new[ii] -= 2*np.pi

            elif x_new[ii] < -np.pi:
                x_new[ii] += 2*np.pi

        self.xv_unwrap = copy(self.xv)
        self.xv[:,0] = x_new

    def F(self, x, v, t):
        g = 9.8
        F = - g/self.l*np.sin(x)

        return F


from .base import BaseProcessor

from collections import defaultdict
from urllib.parse import quote

import logging
import markdown2
import textwrap
import os
import re

logger = logging.getLogger(__name__)

with_pil = False

try:
    from PIL import Image, UnidentifiedImageError
    with_pil = True
except ImportError:
    pass

from jinja2 import Environment, FileSystemLoader, ChoiceLoader, DictLoader

try:
    from pygments import highlight
    from pygments.styles import get_style_by_name
    from pygments.lexers import get_lexer_by_name
    from pygments.formatters import HtmlFormatter
    with_pygments = True
except ImportError:
    with_pygments = False

try:
    from bs4 import BeautifulSoup as bs
    with_bs = True
except ImportError:
    with_bs = False

class JinjaProcessor(BaseProcessor):

    def highlight_styles(self, code, style_name=None):
        if style_name is None:
            style_name = self.site.config.get('pygments', {}).get('style', 'monokai')
        style = get_style_by_name(style_name)
        return '<style type="text/css">{}</style>'.format(HtmlFormatter(style=style).get_style_defs('.highlight .{}'.format(style_name)))

    def highlight(self, code, language='python', style_name=None, strip=True, deindent=True):
        if style_name is None:
            style_name = self.site.config.get('pygments', {}).get('style', 'monokai')
        lexer = get_lexer_by_name(language)
        style = get_style_by_name(style_name)
        if deindent:
            code = textwrap.dedent(code)
        if strip:
            code = code.strip()
        return highlight(code, lexer, HtmlFormatter(style=style, cssclass='{}'.format(style_name)))

    def picture(self, filename, prefs=['webp', 'png', 'jpeg'], sizes='50vw', **kwargs):
        file_path = self.file(filename)
        file_dir = os.path.dirname(file_path)
        if not file_path:
            return ""
        source_path = self.source_path(filename)
        filename = os.path.basename(source_path)
        source_dir = os.path.dirname(source_path)
        basename, zoom, ext = re.match(r"^(.*?)(?:@(\d+x))?\.(.*)$", filename).groups()
        extra_args = []
        for k, v in kwargs.items():
            extra_args.append(f'{k}="{v}"')
        extra_args = " ".join(extra_args)
        files = os.listdir(source_dir)
        alternatives = defaultdict(list)
        for fname in files:
            if re.match("{}(?:@\d+x)?\.\w+$".format(re.escape(basename)), fname):
                _, zoom, ext = re.match(r"^(.*?)(?:@(\d+x))?\.(.*)$", fname).groups()
                if zoom is None:
                    zoom = '1x'
                if with_pil:
                    try:
                        # we determine the exact image width to include
                        image = Image.open(os.path.join(source_dir, fname))
                        zoom = '{}w'.format(image.size[0])
                    except UnidentifiedImageError:
                        # we cannot properly read this image, ignore it
                        logger.warning(f"Cannot read image {source_dir}/{fname}, ignoring its size...")
                        pass
                alternatives[ext].append((fname, zoom))

        html_alternatives = ""

        sizestr = ""
        if sizes:
            sizestr = f'sizes="{sizes}"'

        def generate_alternative(ext, ext_alternatives):

            alts = []
            for f, z in ext_alternatives:
                fp = quote(f"{file_dir}/{f}")
                alts.append(f"{fp} {z}")
            joined_alternatives = ", ".join(alts)
            return f"""
    <source type="image/{ext}" srcset="{joined_alternatives}" {sizestr}>
"""

        # we add the preferred formats first
        for pref in prefs:
            if pref in alternatives:
                html_alternatives += generate_alternative(pref, alternatives[pref])
        # then we add the rest
        for ext, ext_alternatives in alternatives.items():
            if not ext in prefs:
                html_alternatives += generate_alternative(ext, ext_alternatives)
        return f"""
<picture>
    {html_alternatives}
    <img src="{file_path}" {extra_args}>
</picture>
"""

    def markdown(self, text):
        result = markdown2.markdown(text, extras=['footnotes', 'fenced-code-blocks'])
        return result

    def translate(self, key, *args, **kwargs):
        result = super().translate(key, *args, **kwargs)
        translate_hint = self.site.config.get('translate-hint')
        if translate_hint:
            return translate_hint.format(value=result, key=key, args=args, kwargs=kwargs)
        return result

    def jinja(self, vars, input, **kwargs):
        context = vars.copy()
        context.update(kwargs)
        return self.process(input, context)

    def get_jinja_env(self, input, vars):
        dict_loader = DictLoader({'input' : input})
        choice_loader = ChoiceLoader([dict_loader, FileSystemLoader('{}/templates'.format(self.site.src_path)), FileSystemLoader(self.site.src_path)])
        env = Environment(loader=choice_loader)

        # we add some useful filters
        env.filters['href'] = self.href
        env.filters['full_href'] = self.full_href
        env.filters['file'] = self.file
        env.filters['markdown'] = self.markdown
        env.filters['picture'] = self.picture
        env.filters['jinja'] = lambda input, **kwargs: self.jinja(vars, input, **kwargs)
        if with_pygments:
            env.filters['highlight'] = self.highlight
            env.filters['highlight_styles'] = self.highlight_styles
        env.filters['translate'] = self.translate
        for filters in self.site.addons['jinja-filters']:
            for name, f in filters:
                env.filters[name] = f
        return env

    def process(self, input, vars):
        env = self.get_jinja_env(input, vars)
        template = env.get_template('input')
        result = template.render(**vars)
        if with_bs and False:
            soup=bs(result, "html.parser")
            return soup.prettify()
        return result


"""
Developer: Adam M. Terwilliger
Version: March 1, 2019
Purpose: CSE 842 -- HW2
Details: Natural Language Processing course
            at Michigan State University
         Implemention of document classifier
            using DAN and RNN
Dataset: IMDb movie sentiment database
            1108 training docs
"""

import sys
import numpy as np
import torch
import torch.utils.data

from time import time

np.set_printoptions(threshold=np.nan)
np.set_printoptions(suppress=True)

sys.dont_write_bytecode = True

import cse842_hw2_utils as util
from cse842_hw2_dataset import Dataset
from cse842_hw2_dan import dan
from cse842_hw2_rnn import rnn

def main(params):

    num_epochs = 100
    batch_size = 512
    num_layers = 4
    hidden_size = 64
    emb_size = 100
    learning_rate = 0.01
    optim_type = "adam"
    mode = "rnn"

    util.Logger.log("num_epochs = {}".format(num_epochs))
    util.Logger.log("batch_size = {}".format(batch_size))
    util.Logger.log("num_layers = {}".format(num_layers))
    util.Logger.log("hidden_size = {}".format(hidden_size))
    util.Logger.log("emb_size = {}".format(emb_size))
    util.Logger.log("learning_rate = {}".format(learning_rate))
    util.Logger.log("optim_type = {}".format(optim_type))
    util.Logger.log("mode = {}".format(mode))

    random_seed = 7
    util.init_torch(random_seed,random_seed)

    train_dataset = Dataset('train', mode, emb_size)
    val_dataset = Dataset('val', mode, emb_size)
    test_dataset = Dataset('test', mode, emb_size)

    train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=Dataset.collate)
    val_data_loader = torch.utils.data.DataLoader(val_dataset, batch_size=16, shuffle=False, collate_fn=Dataset.collate)

    if mode == "dan":
        model = dan(num_layers, emb_size, hidden_size)
    elif mode == "rnn":
        model = rnn(num_layers, emb_size, hidden_size)

    model = torch.nn.DataParallel(model)

    loss_function = torch.nn.CrossEntropyLoss()
    if optim_type == "sgd":
        optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)
    else:
        optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)

    util.print_weights(model)

    curr_iter = 0
    curr_loss = 0
    loss_iter = len(train_dataset)

    total = 0
    correct = 0

    start_time = time()

    for epoch in range(num_epochs+1):

        for data, labels in train_data_loader:

            # change to training mode
            model.train()

            model.zero_grad()

            pred = model(data)

            loss = loss_function(pred, labels)

            loss.backward()

            optimizer.step()

            # compute accuracy across C=2
            _, cls_pred = torch.max(pred, 1)

            # detach gradient
            # bring back to cpu / numpy
            labels = labels.detach().cpu().numpy()
            cls_pred = cls_pred.detach().cpu().numpy()

            total += len(labels)
            correct += np.sum(labels==cls_pred)

            #util.Logger.log(correct+total+correct/total)

            curr_iter += 1

            # don't need to calc gradient
            curr_loss += loss.detach()

        if epoch % 1 == 0 and epoch > 0:
            time_str, _ = util.compute_eta(start_time, epoch, num_epochs)
            util.Logger.log("-----------------------------------------")
            util.Logger.log("Epoch: {}, ETA: {}".format(epoch, time_str))
            util.Logger.log("Training loss: {:.4f}".format(curr_loss/curr_iter))
            util.Logger.log("Training accuracy: {:.4f}".format(correct/total))
            curr_loss = 0
            curr_iter = 0
            correct = 0
            total = 0

        # testing phase
        if epoch % 1 == 0 and epoch > 0:

            # change to testing mode
            model.eval()

            val_correct = 0
            val_total = 0

            for val_data, val_labels in val_data_loader:

                pred = model(val_data)

                # compute accuracy across C=2
                _, cls_pred = torch.max(pred, 1)

                # detach gradient
                # bring back to cpu / numpy
                val_labels = val_labels.detach().cpu().numpy()
                cls_pred = cls_pred.detach().cpu().numpy()

                val_total += len(val_labels)
                val_correct += np.sum(val_labels==cls_pred)

            util.Logger.log("\nValidation accuracy: {:.4f}".format(val_correct/val_total))

    util.Logger.log("num_epochs = {}".format(num_epochs))
    util.Logger.log("batch_size = {}".format(batch_size))
    util.Logger.log("num_layers = {}".format(num_layers))
    util.Logger.log("hidden_size = {}".format(hidden_size))
    util.Logger.log("emb_size = {}".format(emb_size))
    util.Logger.log("learning_rate = {}".format(learning_rate))
    util.Logger.log("optim_type = {}".format(optim_type))
    util.Logger.log("mode = {}".format(mode))

if __name__ == "__main__":
    main(sys.argv)


import json

class Checkpoint:

    def __init__(self,value,coordinates):
        self.value = value
        self.coordinates = coordinates

    def __lt__(self,other):
        return self.value < other.value

class Map:

    DEFAULT_SETTINGS = '{"wall_char": "#", "passable_char": "."}'

    def __init__(self,walls=None,checkpoints=None,settings=None):
        if walls == None:
            self.walls = [[]]
        else:
            self.walls = walls

        if checkpoints == None:
            self.checkpoints = []
        else:
            self.checkpoints = checkpoints

        if settings == None:
            self.settings = json.loads(Map.DEFAULT_SETTINGS)
        else:
            self.settings = json.loads(settings)

    def __str__(self):
        map_str_arr = []

        for i in range(0,len(self.walls)):
            row_arr = [ self.settings['wall_char'] if x else self.settings['passable_char'] \
                    for x in self.walls[i] ]
            map_str_arr.append(row_arr)

        for checkpoint in self.checkpoints:
            i, j = checkpoint.coordinates
            map_str_arr[i][j] = str(checkpoint.value)

        for i, line in enumerate(map_str_arr):
            map_str_arr[i] = ''.join(x for x in line)

        return '\n'.join(x for x in map_str_arr)

    @staticmethod
    def load_map_from_file(file_name,settings=None):
        map_file = open(file_name,'r')

        # Settings is in JSON format
        # See DEFAULT_SETTINGS for required data
        if settings == None:
            settings_data = json.loads(Map.DEFAULT_SETTINGS)
        else:
            settings_data = json.loads(settings)

        cur_line = map_file.readline()

        cols = len(cur_line) - 1
        rows = 0

        # Rewind file
        map_file.seek(0)

        walls = []
        checkpoints = []

        # Get the unppassable parts of the maps and the checkpoints
        for i, line in enumerate(map_file):

            # Check number of columns
            if len(cur_line) - 1 != cols:
                raise MapLoadingError("Expected {} columns, got {}".format(cols,len(cur_line)), i)

            row_walls = [False] * cols

            for j in range(0,cols):
                row_walls[j] = line[j] == settings_data["wall_char"]

                if line[j].isdigit():
                    checkpoint = Checkpoint(int(line[j]),(i,j))
                    checkpoints.append(checkpoint)

            walls.append(row_walls)
            checkpoints = sorted(checkpoints)

        map_file.close()

        return Map(walls,checkpoints,settings)

class MapError(Exception):
    pass

class MapLoadingError(MapError):

    def __init__(self, message, line):
        self.message = message
        self.line = line


from flask import Flask, render_template, request, jsonify, redirect, url_for
from data import Articles
import pandas as pd
import numpy as np
import re
#from XGBoostModel import *
import RandomForest_webapp as rf
import HTS_analysis as hts_a
import process_data as proc
import roc
from sklearn.ensemble import RandomForestClassifier
from flask import send_from_directory

#!!! pip install Flask-PyMongo
from flask_pymongo import PyMongo
from pymongo import MongoClient
#import urllib.request, json
import os
from werkzeug.utils import secure_filename

UPLOAD_FOLDER = 'uploads/'
ALLOWED_EXTENSIONS = set(['txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif'])
ALLOWED_EXTENSIONS2 = set(['csv'])

app = Flask(__name__)
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
mongo = PyMongo(app)
def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS2

@app.route('/')
def index():
    return render_template('home.html')

@app.route('/register', methods=['POST'])
def register():
    return render_template('home.html')

@app.route('/fraggle', methods=['GET', 'POST'])
def upload_file():
    if request.method == 'POST':
        # check if the post request has the file part
        if 'file' not in request.files:
            flash('No file part')
            return redirect(request.url)
        file = request.files['file']
        # if user does not select file, browser also
        # submit a empty part without filename
        if file.filename == '':
            flash('No selected file')
            return redirect(request.url)
        if file and allowed_file(file.filename):
            filename = secure_filename(file.filename)
            file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
            return redirect(url_for('uploaded_file', filename=filename))
    #Return frggle.html if no file has yet been uploaded
    return render_template('fraggle.html')

@app.route('/uploads/<filename>',methods=['GET', 'POST'])
def uploaded_file(filename):
    if request.method == 'POST':
        # check if the post request has the file part
        if 'file' not in request.files:
            flash('No file part')
            return redirect(request.url)
        file = request.files['file']
        # if user does not select file, browser also
        # submit a empty part without filename
        if file.filename == '':
            flash('No selected file')
            return redirect(request.url)
        if file and allowed_file(file.filename):
            filename = secure_filename(file.filename)
            print(filename)
            file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
            return redirect(url_for('uploaded_HTS', filename = filename))
    elif request.method == 'GET':
        # make a pd.dataframe of training data
        df = pd.read_csv(os.path.join(app.config['UPLOAD_FOLDER'], filename))
        # use all features and yfill (no NaNs, filled with 0)
        features, yfill = proc.features_yfill(df)
        #train test split at 20%
        X_train, X_test, y_train, y_test = rf.train_test_split(features, yfill, test_size=0.20, random_state=1, stratify =yfill)

        #Optional: oversampling of minority class for training purposes
        #X_train_over, y_train_over = proc.oversample(X_train,y_train, r = 0.3)
        #rffit, y_predict = rf.randomforest(X_train_over, X_test, y_train_over, y_test, num_est=50, cls_w = 'balanced_subsample')

        #fit the Random Forest classifier: would like to add in a grid search
        rffit, y_predict = rf.randomforest(X_train.values, X_test, y_train.values, y_test)

        # Use below to run a grid search .... takes to long to work right now
        #rffit, y_predict = rf.randomforest(X_train.values, X_test, y_train.values, y_test, grid_search = 'small')

        #pickle the fit model for use with test data
        proc._pickle(rffit, 'RFC_fit.pkl')

        #set_threshold_recall is a function which determines the threshold to set such that recall is optimized (the median of the available thresholds that return the second best recall (not 1.0))
        precision_list, recall_list, median_recall_index, medianrecall_threshold = rf.set_threshold_recall(rffit, X_train, X_test, y_train, y_test)

        #print_threshold uses the trained model and the selected threshold (in this case recall optimized) to return listed statistics
        precision, recall, fpr, fpr_test, tpr_test, cm = rf.print_threshold(rffit, X_train, X_test, y_train, y_test, medianrecall_threshold)
        r_cm = pd.DataFrame(cm)
        proc._pickle(medianrecall_threshold, 'medianrecall_threshold.pkl')

        #make a pd.dataframe of the stats for display
        recall_opt_stats = pd.DataFrame([[format(medianrecall_threshold, '.2f'),format(recall, '.2f'), format(fpr, '.2f'), format(precision, '.2f'), ]], columns = ['Suggested Threshold','True Positive Rate (Recall)', 'False Positive Rate (Fall-out)','Precision'])

        # repeat the threshold selection process for precision optimization
        p_precision, p_recall, p_median_precision, threshold_precision = rf.set_threshold_precision(rffit, X_train, X_test, y_train, y_test)
        p_precision, p_recall, p_fpr, p_fpr_test, p_tpr_test, p_cm = rf.print_threshold(rffit, X_train, X_test, y_train, y_test, threshold_precision)
        p_cm = pd.DataFrame(p_cm)
        precision_opt_stats = pd.DataFrame([[format(threshold_precision, '.2f'),format(p_recall, '.2f'), format(p_fpr, '.2f'), format(p_precision, '.2f'), ]], columns = ['Suggested Threshold','True Positive Rate (Recall)', 'False Positive Rate (Fall-out)','Precision'])

        #produce a ROC plot
        test_prob = rffit.predict_proba(X_test)
        roc.plot_roc(X_train.values, y_train.values, y_test, test_prob, 'Test', RandomForestClassifier,  max_depth = 10, max_features= 30, min_samples_leaf= 2, min_samples_split = 2)
        feature_description = rf.plot_features(features, rffit, 'Identifier',n=10)
        #option for overfit data
        #roc.plot_roc(X_train_over, y_train_over, y_test, test_prob, 'Test', RandomForestClassifier,  max_depth = 10, max_features= 30, min_samples_leaf= 2, min_samples_split = 2)
        #roc.simple_roc(y_test, test_prob, 'ROC_RFC')
        pd.set_option('display.max_colwidth', -1)

        return render_template("rock.html", data_recall_opt=recall_opt_stats.to_html(index = False, classes = "data_recall_opt"),data_precision_opt=precision_opt_stats.to_html(index = False, classes = "data_precision_opt"), rocname = 'Test',f_descrip = feature_description.to_html(index = False, classes = "f_descrip"), recall_cm = r_cm.to_html(classes = "cm"), precision_cm = p_cm.to_html(classes = "cm"))

#f_descrip = feature_description.to_html(classes = "table table-condensed")
@app.route('/hit')
def uploaded_HTS(filename = 'MTH1_HTS_5500.csv'):
    # Read in HTS data and get features (no scores to model live app)
    df = pd.read_csv(os.path.join(app.config['UPLOAD_FOLDER'], filename))
    features, y, ids = proc.features_yfill_HTS(df)

    rffit = proc._unpickle('RFC_fit.pkl')
    mrecall_threshold = proc._unpickle('medianrecall_threshold.pkl')
    scored_hts = hts_a.score_HTS(rffit, features, ids)
    top50 = scored_hts.iloc[:50,:]

    #Below is for validation purposes only (will only be seen for MTH1 dataset where I have to scored data)
    y_proba = rffit.predict_proba(features)
    roc.plotHTS_roc(features, y, y_proba, 'Test_HTS')
    precision, recall, fpr, fpr_test, tpr_test, cm = hts_a.print_threshold(rffit, features, y, mrecall_threshold)
    cm = pd.DataFrame(cm)
    recall_opt_stats = pd.DataFrame([[format(mrecall_threshold, '.2f'),format(recall, '.2f'), format(fpr, '.2f'), format(precision, '.2f'), ]], columns = ['Suggested Threshold','True Positive Rate (Recall)', 'False Positive Rate (Fall-out)','Precision'])

    return render_template('hit.html', articles = articles, data = recall_opt_stats.to_html(index = False, classes = "table table-condensed"), hits = top50.to_html(index = False, classes = "table table-condensed"), hts_cm = cm.to_html(classes = "table table-condensed") )

    '''@app.route('/uploads/<filename>',methods=['GET', 'POST'])
    def uploaded_HTS(filename):
        return send_from_directory(app.config['UPLOAD_FOLDER'],
                                   filename)
    '''
    '''
    client = MongoClient()
    #set database that collection is in
    db = client.fragments
    #add json call to mongo
    db.fragments.insert_many(df)
    #db.review.update(data, data, upsert=True)
    '''

@app.route('/about')
def articles():
    return render_template('about.html', articles = articles)

if __name__ == '__main__':
    app.run(debug=True)


import aiopg
from sqlalchemy import create_engine, MetaData

from db import user, contact
from settings import config

connection_str = "mysql+pymysql://{user}:{password}@{host}:{port}/{database}"

def create_tables(engine):
    meta = MetaData()
    meta.create_all(bind=engine, tables=[user, contact])

def sample_data(engine):
    conn = engine.connect()
    conn.execute(user.insert(), [
        {'name': 'huyendtp', 'phone': '0909090909'}
    ])
    conn.execute(contact.insert(), [
        {'name': 'huyendtp', 'address': '17 Ba Huyen Thanh Quan', 'user_id': 1},
        {'name': 'huyendtp', 'address': '740/8 Su Van Hanh', 'user_id': 1}
    ])
    conn.close()

if __name__ == '__main__':
    db_url = connection_str.format(**config['mysql'])
    engine = create_engine(db_url)

    create_tables(engine)
    sample_data(engine)


#!/usr/bin/env python
# ROS IMPORTS
import roslib; roslib.load_manifest('simple_ur_driver')
import rospy
import tf; import tf_conversions as tf_c
import PyKDL
# URDF
from urdf_parser_py.urdf import URDF
from pykdl_utils.kdl_kinematics import KDLKinematics
# MSGS and SERVICES
from simple_ur_msgs.srv import *
from sensor_msgs.msg import JointState
from geometry_msgs.msg import PoseStamped
from predicator_msgs.msg import *
from std_msgs.msg import *
import time
import threading
import socket
# URX Universal Robot Driver
import urx
# OTHER
import logging
import numpy as np
from pid import PID

class URDriver():
    MAX_ACC = .5
    MAX_VEL = 1.8
    JOINT_NAMES = ['shoulder_pan_joint', 'shoulder_lift_joint', 'elbow_joint',
               'wrist_1_joint', 'wrist_2_joint', 'wrist_3_joint']

    def __init__(self):
        rospy.init_node('ur_simulation',anonymous=True)
        rospy.logwarn('SIMPLE_UR SIMULATION DRIVER LOADING')
        # TF
        self.broadcaster_ = tf.TransformBroadcaster()
        self.listener_ = tf.TransformListener()
        # SERVICES
        self.servo_to_pose_service = rospy.Service('simple_ur_msgs/ServoToPose', ServoToPose, self.servo_to_pose_call)
        self.set_stop_service = rospy.Service('simple_ur_msgs/SetStop', SetStop, self.set_stop_call)
        self.set_teach_mode_service = rospy.Service('simple_ur_msgs/SetTeachMode', SetTeachMode, self.set_teach_mode_call)
        self.set_servo_mode_service = rospy.Service('simple_ur_msgs/SetServoMode', SetServoMode, self.set_servo_mode_call)
        # PUBLISHERS AND SUBSCRIBERS
        self.driver_status_publisher = rospy.Publisher('/ur_robot/driver_status',String)
        self.robot_state_publisher = rospy.Publisher('/ur_robot/robot_state',String)
        self.joint_state_publisher = rospy.Publisher('joint_states',JointState)
        # self.follow_pose_subscriber = rospy.Subscriber('/ur_robot/follow_goal',PoseStamped,self.follow_goal_cb)
        # Predicator
        self.pub_list = rospy.Publisher('/predicator/input', PredicateList)
        self.pub_valid = rospy.Publisher('/predicator/valid_input', ValidPredicates)
        rospy.sleep(1)
        pval = ValidPredicates()
        pval.pheader.source = rospy.get_name()
        pval.predicates = ['moving', 'stopped', 'running']
        pval.assignments = ['robot']
        self.pub_valid.publish(pval)
        # Rate
        self.run_rate = rospy.Rate(100)
        self.run_rate.sleep()
        ### Set Up Simulated Robot ###
        self.driver_status = 'IDLE'
        self.robot_state = 'POWER OFF'
        robot = URDF.from_parameter_server()
        self.kdl_kin = KDLKinematics(robot, 'base_link', 'ee_link')
        # self.q = self.kdl_kin.random_joint_angles()
        self.q = [-1.5707,-.785,-3.1415+.785,-1.5707-.785,-1.5707,0] # Start Pose?
        self.start_pose = self.kdl_kin.forward(self.q)
        self.F_start = tf_c.fromMatrix(self.start_pose)
        # rospy.logwarn(self.start_pose)
        # rospy.logwarn(type(self.start_pose))
        # pose = self.kdl_kin.forward(q)
        # joint_positions = self.kdl_kin.inverse(pose, q+0.3) # inverse kinematics
        # if joint_positions is not None:
        #     pose_sol = self.kdl_kin.forward(joint_positions) # should equal pose
        # J = self.kdl_kin.jacobian(q)
        # rospy.logwarn('q:'+str(q))
        # rospy.logwarn('joint_positions:'+str(joint_positions))
        # rospy.logwarn('pose:'+str(pose))
        # if joint_positions is not None:
        #     rospy.logwarn('pose_sol:'+str(pose_sol))
        # rospy.logwarn('J:'+str(J))

        ### START LOOP ###
        while not rospy.is_shutdown():
            if self.driver_status == 'TEACH':
                self.update_from_marker()

            # if self.driver_status == 'SERVO':
            #     self.update_follow()

            # Publish and Sleep
            self.publish_status()
            self.send_command()
            self.run_rate.sleep()

        # Finish
        rospy.logwarn('SIMPLE UR - Simulation Finished')

    def update_from_marker(self):
        try:
            F_target_world = tf_c.fromTf(self.listener_.lookupTransform('/world','/endpoint_interact',rospy.Time(0)))
            F_target_base = tf_c.fromTf(self.listener_.lookupTransform('/base_link','/endpoint_interact',rospy.Time(0)))
            F_base_world = tf_c.fromTf(self.listener_.lookupTransform('/world','/base_link',rospy.Time(0)))
            self.F_command = F_base_world.Inverse()*F_target_world
            M_command = tf_c.toMatrix(self.F_command)

            joint_positions = self.kdl_kin.inverse(M_command, self.q) # inverse kinematics
            if joint_positions is not None:
                pose_sol = self.kdl_kin.forward(joint_positions) # should equal pose
                self.q = joint_positions
            else:
                rospy.logwarn('no solution found')

            # self.send_command()

        except (tf.LookupException, tf.ConnectivityException, tf.ExtrapolationException) as e:
            rospy.logwarn(str(e))

    def send_command(self):
        self.current_joint_positions = self.q
        msg = JointState()
        msg.header.stamp = rospy.get_rostime()
        msg.header.frame_id = "simuated_data"
        msg.name = self.JOINT_NAMES
        msg.position = self.current_joint_positions
        msg.velocity = [0]*6
        msg.effort = [0]*6
        self.joint_state_publisher.publish(msg)

        pose = self.kdl_kin.forward(self.q)
        F = tf_c.fromMatrix(pose)
        # F = self.F_command
        self.current_tcp_pose = tf_c.toMsg(F)
        self.current_tcp_frame = F
        self.broadcaster_.sendTransform(tuple(F.p),tuple(F.M.GetQuaternion()),rospy.Time.now(), '/endpoint','/base_link')

    def set_teach_mode_call(self,req):
        if req.enable == True:
            # self.rob.set_freedrive(True)
            self.driver_status = 'TEACH'
            return 'SUCCESS - teach mode enabled'
        else:
            # self.rob.set_freedrive(False)
            self.driver_status = 'IDLE'
            return 'SUCCESS - teach mode disabled'

    def set_servo_mode_call(self,req):
        if req.mode == 'SERVO':
            self.driver_status = 'SERVO'
            return 'SUCCESS - servo mode enabled'
        elif req.mode == 'DISABLE':
            self.driver_status = 'IDLE'
            return 'SUCCESS - servo mode disabled'

    def set_stop_call(self,req):
        rospy.logwarn('SIMPLE UR - STOPPING ROBOT')
        self.rob.stop()
        return 'SUCCESS - stopped robot'

    def servo_to_pose_call(self,req):
        if self.driver_status == 'SERVO':
            rospy.logwarn(req)
            self.F_command = tf_c.fromMsg(req.target)
            M_command = tf_c.toMatrix(self.F_command)
            # Calculate IK
            joint_positions = self.kdl_kin.inverse(M_command, self.q) # inverse kinematics
            if joint_positions is not None:
                pose_sol = self.kdl_kin.forward(joint_positions) # should equal pose
                self.q = joint_positions
            else:
                rospy.logwarn('no solution found')
            # self.send_command(F_command)
            return 'SUCCESS - moved to pose'
        else:
            rospy.logerr('SIMPLE UR -- Not in servo mode')
            return 'FAILED - not in servo mode'

    def publish_status(self):
        self.driver_status_publisher.publish(String(self.driver_status))
        self.robot_state_publisher.publish(String(self.robot_state))

        ps = PredicateList()
        ps.pheader.source = rospy.get_name()
        ps.statements = []

        statement = PredicateStatement( predicate='moving',
                                        confidence=1,
                                        value=PredicateStatement.FALSE,
                                        num_params=1,
                                        params=['robot', '', ''])
        ps.statements += [statement]
        statement = PredicateStatement( predicate='stopped',
                                        confidence=1,
                                        value=PredicateStatement.TRUE,
                                        num_params=1,
                                        params=['robot', '', ''])
        ps.statements += [statement]
        statement = PredicateStatement( predicate='running',
                                        confidence=1,
                                        value=PredicateStatement.TRUE,
                                        num_params=1,
                                        params=['robot', '', ''])
        ps.statements += [statement]
        self.pub_list.publish(ps)

    def check_robot_state(self):
        self.robot_state == 'RUNNING SIMULATION'
        self.driver_status = 'IDLE - SIMULATION'

if __name__ == "__main__":
    robot_driver = URDriver()


#import libraries of python opencv
import cv2
import numpy as np
import datetime
import math

#globals
width = 0
height = 0
EntranceCounter = 0
ExitCounter = 0
LineOffsets = 150

#Check if an object is entering in monitored zone
def CheckEntranceLineCrossing(y, CoorYEntranceLine, CoorYExitLine):
  AbsDistance = abs(y - CoorYEntranceLine)

  if ((AbsDistance <= 2) and (y < CoorYExitLine)):
        return 1
  else:
        return 0

#Check if an object is exiting from monitored zone
def CheckExitLineCrossing(y, CoorYEntranceLine, CoorYExitLine):
    AbsDistance = abs(y - CoorYExitLine)

    if ((AbsDistance <= 2) and (y > CoorYEntranceLine)):
        return 1
    else:
        return 0

#create VideoCapture object and read from video file
cap = cv2.VideoCapture(0)
#use trained cars XML classifiers
car_cascade = cv2.CascadeClassifier('cars.xml')

#read until video is completed
while True:
    #capture frame by frame
    ret, frame = cap.read()

    height = int(np.size(frame,0))
    width = int(np.size(frame,1))

    #This same thing can be done by modifying the while statement
    if not ret:
        continue

    #convert video into gray scale of each frames
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    #Draw entrance and exit lines
    CoorYEntranceLine = int((height / 2)-LineOffsets)
    CoorYExitLine = int((height / 2)+LineOffsets)
    cv2.line(frame, (0,CoorYEntranceLine), (width,CoorYEntranceLine), (255, 0, 0), int(2), int(1))
    cv2.line(frame, (0,CoorYExitLine), (width,CoorYExitLine), (0, 0, 255), int(2), int(1))

    #detect cars in the video
    cars = car_cascade.detectMultiScale(gray, 1.1, 3)

    #to draw arectangle in each cars
    for (x,y,w,h) in cars:
        cv2.rectangle(frame,(x,y),(x+w,y+h),(0,255,0),2)

        #find object's centroid
        CoordXCentroid = int((x+x+w)/2)
        CoordYCentroid = int((y+y+h)/2)
        ObjectCentroid = (CoordXCentroid,CoordYCentroid)
        cv2.circle(frame, ObjectCentroid, 1, (0, 0, 0), 5)

        if (CheckEntranceLineCrossing(CoordYCentroid,CoorYEntranceLine,CoorYExitLine)):
            EntranceCounter += 1

        if (CheckExitLineCrossing(CoordYCentroid,CoorYEntranceLine,CoorYExitLine)):
            ExitCounter += 1

    #Write entrance and exit counter values on frame and shows it
    cv2.putText(frame, "Entrances: {}".format(str(EntranceCounter)), (10, 50), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (250, 0, 1), 2)
    cv2.putText(frame, "Exits: {}".format(str(ExitCounter)), (10, 70), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 2)
    #display the resulting frame
    cv2.imshow('video', frame)
    #press Q on keyboard to exit
    if cv2.waitKey(25) & 0xFF == ord('q'):
        break
#release the videocapture object
cap.release()
#close all the frames
cv2.destroyAllWindows()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
from django.conf import settings
import vlkjsonfield.fields

class Migration(migrations.Migration):

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
    ]

    operations = [
        migrations.CreateModel(
            name='Answer',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('answer', models.IntegerField()),
                ('created', models.DateTimeField(auto_now_add=True)),
                ('answered_by', models.ForeignKey(related_name=b'answers', to=settings.AUTH_USER_MODEL)),
            ],
            options={
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='Question',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('question', models.TextField()),
                ('answer_type', models.PositiveSmallIntegerField(choices=[(1, b'Numeric'), (2, b'Boolean'), (3, b'Choice')])),
                ('definitions', vlkjsonfield.fields.VLKJSONField(default={})),
            ],
            options={
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='SLA',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('name', models.CharField(max_length=127)),
                ('expires', models.DateTimeField(null=True, blank=True)),
                ('due_date', models.DateTimeField(null=True, blank=True)),
            ],
            options={
            },
            bases=(models.Model,),
        ),
        migrations.AddField(
            model_name='question',
            name='sla',
            field=models.ForeignKey(related_name=b'questions', to='slas.SLA'),
            preserve_default=True,
        ),
        migrations.AddField(
            model_name='answer',
            name='question',
            field=models.ForeignKey(related_name=b'answers', to='slas.Question'),
            preserve_default=True,
        ),
    ]


import boto3
from boto3.s3.transfer import TransferConfig

s3_client = boto3.client('s3')

S3_BUCKET = 'mybucket'
FILE_PATH = '/path/to/file/'
KEY_PATH = "/path/to/s3key/"

def uploadFileS3(filename):
    config = TransferConfig(multipart_threshold=1024*25, max_concurrency=10,
                        multipart_chunksize=1024*25, use_threads=True)
    file = FILE_PATH + filename
    key = KEY_PATH + filename
    s3_client.upload_file(file, S3_BUCKET, key,
    ExtraArgs={ 'ACL': 'public-read', 'ContentType': 'video/mp4'},
    Config = config,
    Callback=ProgressPercentage(file)
    )

uploadFileS3('upload.mp4')


import numpy as np
import root_numpy
import pandas as pd
import sys
from os import path
from sklearn.preprocessing import OneHotEncoder, MinMaxScaler
from sklearn.model_selection import train_test_split
from keras.models import Sequential, Model
from keras.layers import *
from keras.optimizers import SGD, Adam
from keras.activations import relu
# np.set_printoptions(threshold=sys.maxsize)
from keras import layers
from sklearn.utils import shuffle

def prepareTrainingSet( sample , model_vars, label_dict ):
	data = sample[ model_vars ]
#	print(data)
	data = shuffle(data)
	target = data['Muon_genPdgId']
	target = abs(target)
	data = data.drop(columns='Muon_genPdgId')

	pt = data['Muon_pt']
#	data = data.drop(columns='Muon_pt')

	target = target.map(label_dict)
	norm = MinMaxScaler()
	data = norm.fit_transform(data)

	x_train, x_test, y_train, y_test = train_test_split(data, target, test_size = .35, random_state=1)
	pt_train, pt_test, _, _ = train_test_split(pt,target, test_size= .35, random_state=1)

	# x_train = x_train.to_numpy()
	# x_test = x_test.to_numpy()
	y_train = y_train.to_numpy()
	y_test = y_test.to_numpy()

	y_train = np.array([np.array(i) for i in y_train])
	y_test = np.array([np.array(i) for i in y_test])
	pt_train = pt_test.to_numpy()
	pt_test = pt_test.to_numpy()

	return x_train, x_test, y_train, y_test, pt_train, pt_test

def reportSample( sample):
                for i in range(len(sample)):
                        print("Sample "+str(i)+" Report")
                        print( sample[i].shape )

def expandList( df, columnNames):
                outDf = pd.DataFrame()
                for col in columnNames:
                        arr = pd.Series(np.array([j for i in df[col] for j in i]))
                        outDf[col] = arr
                return outDf

def makeData(filename,definedIds,treeName):
	# treeName = 'Events'
	# gPath = '/home/t3-ku/mlazarov/softMVA/CMSSW_10_6_11_patch1/src/KUSoftMVA/MuonAnalysis/test/OutputFiles/'
	data = root_numpy.root2array(filename+'.root',treeName)
	data = pd.DataFrame(data)
	#make gen pdg ID labels for reco muons
	#-999 if unmatched
	pdgIds = np.array([-999 if mu == -1 else data['GenPart_pdgId'][i][mu] for i, idxs in enumerate(data['Muon_genPartIdx']) for j, mu in enumerate(idxs)])
	#get rid of 0 muon events
	data = data.drop([i for i, nMu in enumerate(data['nMuon']) if nMu == 0])
	#expand the list so each row is a muon (not an event)
	muonMask = data.columns.str.contains('Muon_.*')
	expCols = data.loc[:,muonMask].columns
	data = expandList(data, expCols)
	#add in gen pgdIds and jet btags of reco muons
	data['Muon_genPdgId'] = pdgIds
	#drop muons with pt < 2
	data = data.drop([i for i, pt in enumerate(data['Muon_pt']) if pt < 2])
	data = data.reset_index()
	#drop muons matched to anything not in our defined classes: 13, 221, 321, 2211, or 999
	data = data.drop([i for i, ID in enumerate(data['Muon_genPdgId']) if abs(ID) not in definedIds])
	data = data.reset_index()
	# df = df.drop([i for i, ID in enumerate(df['genPdgId']) if abs(ID) not in definedIds])
	data.to_csv(filename+'.csv')
	return data

class DATA:

	def __init__(self,fname,name):
		definedIds = np.array([13,211,321,2212,999])
		self.name = name
		self.treeName = 'Events'
		self.fname = fname
		print(self.fname)
		#speeds up processing time
		if path.exists(self.fname+'.csv'):
			self.data = pd.read_csv(self.fname+'.csv')
		else:
			self.data = makeData(self.fname,definedIds,self.treeName)
		# self.tmp = root_numpy.root2array(self.fname,self.treeName)
		# self.data = pd.DataFrame(self.tmp)

		# self.pdgIds = [-999 if mu == -1 else self.data['GenPart_pdgId'][i][mu] for i, idxs in enumerate(self.data['Muon_genPartIdx']) for j, mu in enumerate(idxs)]
		# self.pdgIds = np.array(self.pdgIds)
		# self.data = self.data.drop([i for i, nMu in enumerate(self.data['nMuon']) if nMu == 0])

		# self.muonMask = self.data.columns.str.contains('Muon_.*')
		# self.expCols = self.data.loc[:,self.muonMask].columns

		# self.data = expandList(self.data, self.expCols)

		# self.data['Muon_genPdgId'] = self.pdgIds

		self.data1 = self.data[abs(self.data.Muon_genPdgId) == 13]
		self.data2 = self.data[self.data.Muon_genPdgId == -999]
		self.data3 = self.data[abs(self.data.Muon_genPdgId) == 11]
		self.data4 = self.data[abs(self.data.Muon_genPdgId) == 211]
		self.data5 = self.data[abs(self.data.Muon_genPdgId) == 321]
		self.data6 = self.data[abs(self.data.Muon_genPdgId) == 2212]

		self.datacoll = {'mu': self.data1, 'U':self.data2, 'e':self.data3, 'pi':self.data4, 'k':self.data5,'p':self.data6}

	def report(self):
		print("Report for data "+self.name)
		print("mu", self.data1.shape[0])
		print("unmatched", self.data2.shape[0])
		print("elec", self.data3.shape[0])
		print("pion", self.data4.shape[0])
		print("kaon", self.data5.shape[0])
		print("prot", self.data6.shape[0])

		print('Relative Frequencies of Classes (total):')
		print(abs(self.data.Muon_genPdgId).value_counts(normalize=True))

	def sample(self, keys, nsamples):
		return [ self.datacoll[k][:s] for i, (k,s) in enumerate(zip(keys,nsamples)) ]

##