'''
Implements a UI for neovim  using tkinter.

* The widget has lines updated/deleted so that any
  given time it only contains what is being displayed.

* The widget is filled with spaces
'''

import sys
import math
import time
from neovim import attach

# from tkquick.gui.tools import rate_limited

from pytknvim.ui_bridge import UIBridge
from pytknvim.screen import Screen
from pytknvim.util import _stringify_key, _stringify_color
from pytknvim.util import _split_color, _invert_color
from pytknvim.util import debug_echo
from pytknvim.util import attach_headless, attach_child
from pytknvim import tk_util

try:
    import Tkinter as tk
    import tkFont as tkfont
    import ttk
except ImportError:
    import tkinter as tk
    import tkinter.font as tkfont

import attr

RESIZE_DELAY = 0.04

def parse_tk_state(state):
    if state & 0x4:
        return 'Ctrl'
    elif state & 0x8:
        return 'Alt'
    elif state & 0x1:
        return 'Shift'

tk_modifiers = ('Alt_L', 'Alt_R',
                'Control_L', 'Control_R',
                'Shift_L', 'Shift_R',
                'Win_L', 'Win_R')

KEY_TABLE = {
    'slash': '/',
    'backslash': '\\',
    'asciicircumf': '^',
    'at': '@',
    'numbersign': '#',
    'dollar': '$',
    'percent': '%',
    'ampersand': '&',
    'asterisk': '*',
    'parenleft': '(',
    'parenright': ')',
    'underscore': '_',
    'plus': '+',
    'minus': '-',
    'bracketleft': '[',
    'bracketright': ']',
    'braceleft': '{',
    'braceright': '}',
    'quotedbl': '"',
    'apostrophe': "'",
    'less': "<",
    'greater': ">",
    'comma': ",",
    'period': ".",
    'BackSpace': 'BS',
    'Return': 'CR',
    'Escape': 'Esc',
    'Delete': 'Del',
    'Next': 'PageUp',
    'Prior': 'PageDown',
    'Enter': 'CR',
}

class MixTk():
    '''
    Tkinter actions we bind and use to communicate to neovim
    '''
    def tk_key_pressed(self,event, **k):
        keysym = event.keysym
        state = parse_tk_state(event.state)
        if event.char not in ('', ' ') \
                    and state in (None, 'Shift'):
            if event.keysym_num == ord(event.char):
                # Send through normal keys
                self._bridge.input(event.char)
                return
        if keysym in tk_modifiers:
            # We don't need to track the state of modifier bits
            return
        if keysym.startswith('KP_'):
            keysym = keysym[3:]

        # Translated so vim understands
        input_str = _stringify_key( KEY_TABLE.get(keysym, keysym), state)
        self._bridge.input(input_str)

    def _tk_quit(self, *args):
        self._bridge.exit()

    # @rate_limited(1/RESIZE_DELAY, mode='kill')
    def _tk_resize(self, event):
        '''Let Neovim know we are changing size'''
        cols = int(math.floor(event.width / self._colsize))
        rows = int(math.floor(event.height / self._rowsize))
        if self._screen.columns == cols:
            if self._screen.rows == rows:
                return
        self.current_cols = cols
        self.current_rows = rows
        self._bridge.resize(cols, rows)
        if self.debug_echo:
            print('resizing c, r, w, h',
                    cols,rows, event.width, event.height)

    def bind_resize(self):
        '''
        after calling,
        widget changes will now be passed along to neovim
        '''
        print('binding resize to', self, self.text)
        self._configure_id = self.text.bind('<Configure>', self._tk_resize)

    def unbind_resize(self):
        '''
        after calling,
        widget size changes will not be passed along to nvim
        '''
        print('unbinding resize from', self)
        self.text.unbind('<Configure>', self._configure_id)

    def _get_row(self, screen_row):
        '''change a screen row to a tkinter row,
        defaults to screen.row'''
        if screen_row is None:
            screen_row = self._screen.row
        return screen_row + 1

    def _get_col(self, screen_col):
        '''change a screen col to a tkinter row,
        defaults to screen.col'''
        if screen_col is None:
            screen_col = self._screen.col
        return screen_col

    def tk_delete_line(self, screen_col=None, screen_row=None,
                                       del_eol=False, count=1):
        '''
        To specifiy where to start the delete from
        screen_col (defualts to screen.row)
        screen_row (defaults to screen.col)

        To delete the eol char aswell
        del_eol (defaults to False)

        count is the number of lines to delete
        '''
        line = self._get_row(screen_row)
        col = self._get_col(screen_col)
        start = "%d.%d" % (line, col)
        if del_eol:
            end = "%d.0" % (line + count)
        else:
            end = "%d.end" % (line + count - 1)
        self.text.delete(start, end)
        gotten = self.text.get(start, end)
        if self.debug_echo == True:
            print('deleted  from ' + start + ' to end ' +end)
            print('deleted '+repr(gotten))

    def tk_pad_line(self, screen_col=None, add_eol=False,
                                    screen_row=None, count=1):
        '''
        add required blank spaces at the end of the line
        can apply action to multiple rows py passing a count
        in
        '''
        line = self._get_row(screen_row)
        col = self._get_col(screen_col)
        for n in range(0, count):
            start = "%d.%d" % (line + n, col)
            spaces = " " * (self.current_cols - col)
            if add_eol:
                spaces += '\n'
            if self.debug_echo:
                pass
                # print('padding from ', start, ' with %d: '
                                                # % len(spaces))
                # print(repr(spaces))
            self.text.insert(start, spaces)

    def _start_blinking(self):
        # cursor is drawn seperatley in the window
        row, col = self._screen.row, self._screen.col
        text, attrs = self._screen.get_cursor()
        pos = "%d.%d" % (row +1, col)

        if not attrs:
            attrs = self._get_tk_attrs(None)
        fg = attrs[1].get('foreground')
        bg = attrs[1].get('background')
        try:
            self.text.stop_blink()
        except Exception:
            pass
        self.text.blink_cursor(pos, fg, bg)

class NvimHandler(MixTk):
    '''These methods get called by neovim'''

    def __init__(self, text, toplevel, address=-1, debug_echo=False):
        self.text = text
        self.toplevel = toplevel
        self.debug_echo = debug_echo

        self._insert_cursor = False
        self._screen = None
        self._foreground = -1
        self._background = -1
        self._pending = [0,0,0]
        self._attrs = {}
        self._reset_attrs_cache()
        self._colsize = None
        self._rowsize = None

        # Have we connected to an nvim instance?
        self.connected = False
        # Connecition Info for neovim
        self.address = address
        cols = 80
        rows = 24
        self.current_cols = cols
        self.current_rows = rows

        self._screen = Screen(cols, rows)
        self._bridge = UIBridge()

    @debug_echo
    def connect(self, *nvim_args, address=None, headless=False, exec_name='nvim'):
        # Value has been set, otherwise default to this functions default value
        if self.address != -1 and not address:
            address = self.address

        if headless:
            nvim = attach_headless(nvim_args, address)
        elif address:
            nvim = attach('socket', path=address, argv=nvim_args)
        else:
            nvim = attach_child(nvim_args=nvim_args, exec_name=exec_name)

        self._bridge.connect(nvim, self.text)
        self._screen = Screen(self.current_cols, self.current_rows)
        self._bridge.attach(self.current_cols, self.current_rows, rgb=True)
        # if len(sys.argv) > 1:
            # nvim.command('edit ' + sys.argv[1])
        self.connected = True
        self.text.nvim = nvim
        return nvim

    @debug_echo
    def _nvim_resize(self, cols, rows):
        '''Let neovim update tkinter when neovim changes size'''
        # TODO
        # Make sure it works when user changes font,
        # only can support mono font i think..
        self._screen = Screen(cols, rows)

    @debug_echo
    def _nvim_clear(self):
        '''
        wipe everyything, even the ~ and status bar
        '''
        self._screen.clear()

        self.tk_delete_line(del_eol=True,
                            screen_row=0,
                            screen_col=0,
                            count=self.current_rows)
        # Add spaces everywhere
        self.tk_pad_line(screen_row=0,
                         screen_col=0,
                         count=self.current_rows,
                         add_eol=True,)

    @debug_echo
    def _nvim_eol_clear(self):
        '''
        delete from index to end of line,
        fill with whitespace
        leave eol intact
        '''
        self._screen.eol_clear()
        self.tk_delete_line(del_eol=False)
        self.tk_pad_line(screen_col=self._screen.col,
                         add_eol=False)

    @debug_echo
    def _nvim_cursor_goto(self, row, col):
        '''Move gui cursor to position'''
        self._screen.cursor_goto(row, col)
        self.text.see("1.0")

    @debug_echo
    def _nvim_busy_start(self):
        self._busy = True

    @debug_echo
    def _nvim_busy_stop(self):
        self._busy = False

    @debug_echo
    def _nvim_mouse_on(self):
        self.mouse_enabled = True

    @debug_echo
    def _nvim_mouse_off(self):
        self.mouse_enabled = False

    @debug_echo
    def _nvim_mode_change(self, mode):
        self._insert_cursor = mode == 'insert'

    @debug_echo
    def _nvim_set_scroll_region(self, top, bot, left, right):
        self._screen.set_scroll_region(top, bot, left, right)

    @debug_echo
    def _nvim_scroll(self, count):
        self._flush()
        self._screen.scroll(count)
        abs_count = abs(count)
        # The minus 1 is because we want our tk_* functions
        # to operate on the row passed in
        delta = abs_count - 1
        # Down
        if count > 0:
            delete_row = self._screen.top
            pad_row = self._screen.bot - delta
        # Up
        else:
            delete_row = self._screen.bot - delta
            pad_row = self._screen.top

        self.tk_delete_line(screen_row=delete_row,
                            screen_col=0,
                            del_eol=True,
                            count=abs_count)
        self.tk_pad_line(screen_row=pad_row,
                         screen_col=0,
                         add_eol=True,
                         count=abs_count)
        # self.text.yview_scroll(count, 'units')

    # @debug_echo
    def _nvim_highlight_set(self, attrs):
        self._attrs = self._get_tk_attrs(attrs)

    # @debug_echo
    def _reset_attrs_cache(self):
        self._tk_text_cache = {}
        self._tk_attrs_cache = {}

    @debug_echo
    def _get_tk_attrs(self, attrs):
        key = tuple(sorted((k, v,) for k, v in (attrs or {}).items()))
        rv = self._tk_attrs_cache.get(key, None)
        if rv is None:
            fg = self._foreground if self._foreground != -1\
                                                else 0
            bg = self._background if self._background != -1\
                                                else 0xffffff
            n = {'foreground': _split_color(fg),
                'background': _split_color(bg),}
            if attrs:
                # make sure that fg and bg are assigned first
                for k in ['foreground', 'background']:
                    if k in attrs:
                        n[k] = _split_color(attrs[k])
                for k, v in attrs.items():
                    if k == 'reverse':
                        n['foreground'], n['background'] = \
                            n['background'], n['foreground']
                    elif k == 'italic':
                        n['slant'] = 'italic'
                    elif k == 'bold':
                        n['weight'] = 'bold'
                        # TODO
                        # if self._bold_spacing:
                            # n['letter_spacing'] \
                                    # = str(self._bold_spacing)
                    elif k == 'underline':
                        n['underline'] = '1'
            c = dict(n)
            c['foreground'] = _invert_color(*_split_color(fg))
            c['background'] = _invert_color(*_split_color(bg))
            c['foreground'] = _stringify_color(*c['foreground'])
            c['background'] = _stringify_color(*c['background'])
            n['foreground'] = _stringify_color(*n['foreground'])
            n['background'] = _stringify_color(*n['background'])
            # n = normal, c = cursor
            rv = (n, c)
            self._tk_attrs_cache[key] = (n, c)
        return rv

    # @debug_echo
    def _nvim_put(self, text):
        '''
        put a charachter into position, we only write the lines
        when a new row is being edited
        '''
        if self._screen.row != self._pending[0]:
            # write to screen if vim puts stuff on  a new line
            self._flush()

        self._screen.put(text, self._attrs)
        self._pending[1] = min(self._screen.col - 1,
                               self._pending[1])
        self._pending[2] = max(self._screen.col,
                               self._pending[2])

    # @debug_echo
    def _nvim_bell(self):
        pass

    # @debug_echo
    def _nvim_visual_bell(self):
        pass

    # @debug_echo
    def _nvim_update_fg(self, fg):
        self._foreground = fg
        self._reset_attrs_cache()
        foreground = self._get_tk_attrs(None)[0]['foreground']
        self.text.config(foreground=foreground)

    # @debug_echo
    def _nvim_update_bg(self, bg):
        self._background = bg
        self._reset_attrs_cache()
        background = self._get_tk_attrs(None)[0]['background']
        self.text.config(background=background)

    # @debug_echo
    def _nvim_update_suspend(self, arg):
        self.root.iconify()

    # @debug_echo
    def _nvim_set_title(self, title):
        self.root.title(title)

    # @debug_echo
    def _nvim_set_icon(self, icon):
        self._icon = tk.PhotoImage(file=icon)
        self.root.tk.call('wm', 'iconphoto',
                          self.root._w, self._icon)

    # @debug_echo
    def _flush(self):
        row, startcol, endcol = self._pending
        self._pending[0] = self._screen.row
        self._pending[1] = self._screen.col
        self._pending[2] = self._screen.col
        if startcol == endcol:
            #print('startcol is endcol return, row %s col %s'% (self._screen.row, self._screen.col))
            return
        ccol = startcol
        buf = []
        bold = False
        for _, col, text, attrs in self._screen.iter(row,
                                    row, startcol, endcol - 1):
            newbold = attrs and 'bold' in attrs[0]
            if newbold != bold or not text:
                if buf:
                    self._draw(row, ccol, buf)
                bold = newbold
                buf = [(text, attrs,)]
                ccol = col
            else:
                buf.append((text, attrs,))
        if buf:
            self._draw(row, ccol, buf)
        else:
            pass
            # print('flush with no draw')

    @debug_echo
    def _draw(self, row, col, data):
        '''
        updates a line :)
        '''
        for text, attrs in data:
            try:
                start = end
            except UnboundLocalError:
                start = "{}.{}".format(row + 1, col)
            end = start+'+{0}c'.format(len(text))

            if not attrs:
                attrs = self._get_tk_attrs(None)
            attrs = attrs[0]

            # if self.debug_echo:
                # print('replacing ', repr(self.text.get(start, end)))
                # print('with ', repr(text), ' at ', start, ' ',end)
            self.text.replace(start, end, text)

            if attrs:
                self.text.apply_attribute(attrs, start, end)
            start

    @debug_echo
    def _nvim_exit(self, arg):
        print('in exit')
        import pdb;pdb.set_trace()
        # self.root.destroy()

    @debug_echo
    def _nvim_update_sp(self, *args):
        pass

class NvimTk(tk_util.Text):
    '''namespace for neovim related methods,
    requests are generally prefixed with _tk_,
    responses are prefixed with _nvim_
    '''
    # we get keys, mouse movements inside tkinter, using binds,
    # These binds are handed off to neovim using _input

    # Neovim interpruts the actions and calls certain
    # functions which are defined and implemented in tk

    # The api from neovim does stuff line by line,
    # so each callback from neovim produces a series
    # of miniscule actions which in the end updates a line

    # So we can shutdown the neovim connections
    instances = []

    def __init__(self, parent, *_, address=False, toplevel=False, **kwargs):
        '''
        :parent: normal tkinter parent or master of the widget
        :toplevel: , if true will resize based off the toplevel etc
        :address: neovim connection info
            named pipe /tmp/nvim/1231
            tcp/ip socket 127.0.0.1:4444
            'child'
            'headless'
        :kwargs: config options for text widget
        '''
        tk_util.Text.__init__(self, parent, **kwargs)
        self.nvim_handler = NvimHandler(text=self,
                                        toplevel=toplevel,
                                        address=address,
                                        debug_echo=False)

        # TODO weak ref?
        NvimTk.instances.append(self)

    def _nvimtk_config(self, *args):
        '''required config'''
        # Hide tkinter cursor
        self.config(insertontime=0)

        # Remove Default Bindings and what happens on insert etc
        bindtags = list(self.bindtags())
        bindtags.remove("Text")
        self.bindtags(tuple(bindtags))

        self.bind('<Key>', self.nvim_handler.tk_key_pressed)

        self.bind('<Button-1>', lambda e: self.focus_set())

        # The negative number makes it pixels instead of point sizes
        size = self.make_font_size(13)
        self._fnormal = tkfont.Font(family='Monospace', size=size)
        self._fbold = tkfont.Font(family='Monospace', weight='bold', size=size)
        self._fitalic = tkfont.Font(family='Monospace', slant='italic', size=size)
        self._fbolditalic = tkfont.Font(family='Monospace', weight='bold',
                                 slant='italic', size=size)
        self.config(font=self._fnormal, wrap=tk.NONE)

        self.nvim_handler._colsize = self._fnormal.measure('M')
        self.nvim_handler._rowsize = self._fnormal.metrics('linespace')

    def nvim_connect(self, *a, **k):
        ''' force connection to neovim '''
        self.nvim_handler.connect(*a, **k)
        self._nvimtk_config()

    @staticmethod
    def kill_all():
        ''' Kill all the neovim connections '''
        raise NotImplementedError
        for self in NvimTk.instances:
            if self.nvim_handler.connected:
                # Function hangs us..
                # self.after(1, self.nvim_handler._bridge.exit)
                self.nvim_handler._bridge.exit()

    def pack(self, *arg, **kwarg):
        ''' connect to neovim if required'''
        tk_util.Text.pack(self, *arg, **kwarg)
        if not self.nvim_handler.connected:
            self.nvim_connect()

        self.nvim_handler.bind_resize()

    def grid(self, *arg, **kwarg):
        ''' connect to neovim if required'''
        tk_util.Text.grid(self, *arg, **kwarg)
        if not self.nvim_handler.connected:
            self.nvim_connect()

        self.nvim_handler.bind_resize()

    def schedule_screen_update(self, apply_updates):
        '''This function is called from the bridge,
           apply_updates calls the required nvim actions'''
        # if time.time() - self.start_time > 1:
            # print()
        # self.start_time = time.time()
        def do():
            apply_updates()
            self.nvim_handler._flush()
            self.nvim_handler._start_blinking()
        self.master.after_idle(do)

    def quit(self):
        ''' destroy the widget, called from the bridge'''
        self.after_idle(self.destroy)

# if __name__ == '__main__':
    # main()


# -*- coding: utf-8 -*-
# @Time : 2022/5/5 10:30
# @Author : Chenglong She
# @File : produce_data.py
# @Software: PyCharm
# 将原图以及labelme生成的mask这两类png图片以112为步距切割成若干个224x224的子图，
# 并删除切割后没有mask存在的标签子图，以及对应的ssdna原图子图

import cv2
import numpy as np
import os
#切割图片，图片文件夹中不能含有其他文件（如json、txt）
file_path = "./img"
height = 224
width = 224
# 遍历目标图像路径
# global root, dirs, files, ssdna_names
directory = list(os.walk(file_path))
root = directory[0][0]
files = directory[0][2]
ssdna_names = []
for name in files:
    if name.split('.')[0][-5:] != 'masks':
        name = name.split('.')[0]
        ssdna_names.append(name)

# 创建不同的id文件夹以便后续分割后的图像存放
for id in ssdna_names:
    folder = os.path.exists(root+'//'+id)
    if not folder:
        os.mkdir(root+'//'+id)

dirs = directory[0][1]

# 对file_path下的所有图像进行分割
for pic_name in files:
    # 分割的图片的位置,图片保存位置
    pic_path = root + '//' + pic_name
    pic_target = root + '//' + pic_name[:9]# 存放分割数据的文件夹
    # 要分割的子图尺寸
    cut_height = height
    cut_width = width

    # 读取要分割的图片，以及其尺寸等数据,cv2.IMREAD_ANYDEPTH可以读进灰度图
    picture = cv2.imread(pic_path, cv2.IMREAD_ANYDEPTH)
    (pic_height, pic_width) = picture.shape

    # 计算能够被cut_height, cut_width整除的填充pixel数量
    difference_h = cut_height - (pic_height % cut_height)
    difference_w = cut_width - (pic_width % cut_width)

    # 对原图进行填充（仅对difference_h，difference_w都为偶数的情况有效）
    after_pad = np.pad(picture,
                       ((int(difference_h), 0),
                        (int(difference_w), 0)),
                       'constant', constant_values=-0)

    # 计算可以划分的横纵的个数
    (after_pad_height, after_pad_width) = after_pad.shape
    num_height = int(after_pad_height / cut_height) * 2 - 1
    num_width = int(after_pad_width / cut_width) * 2 - 1
    step_length = int(cut_width / 2)# 以步长为剪裁框大小的一半进行滑动剪裁

    # for循环迭代生成
    for h in range(0, num_height):
        for w in range(0, num_width):
            h_start_pix = h * step_length
            w_start_pix = w * step_length
            pic = after_pad[h_start_pix: h_start_pix + cut_height,
                  w_start_pix: w_start_pix + cut_width
                  ]
            if 'masks' in pic_name:
                result_path = pic_target + '//' + '{}_{}_{}_{}.png'.format(pic_name.split('.')[0][:9],
                                                                           h + 1, w + 1, 'masks')
            else:
                result_path = pic_target + '//' + '{}_{}_{}.png'.format(pic_name.split('.')[0], h + 1, w + 1)
            cv2.imwrite(result_path, pic)
    print("{} is done".format(pic_name))

# 开始删除子图中没有mask的png图像
directory = list(os.walk(root))
dir_list = directory[0][1]
for dir in dir_list:
    dir_path = root + '//' + dir
    directory_seg = list(os.walk(dir_path))
    root_seg = directory_seg[0][0]
    files_seg = directory_seg[0][2]
    delete_id = []
    for id in files_seg:
        path_seg = root_seg + '//' + id
        if 'masks' in id:
            picture_seg = cv2.imread(path_seg, cv2.IMREAD_ANYDEPTH)
            if picture_seg.max() == 0:
                name_list = id.split('.')[0].split('_')
                name_list.remove('masks')
                name = '_'.join(name_list)
                delete_id.append(name)
    for name in delete_id:
        mask_path = root_seg + '//' + name + "_masks.png"
        ssdna_path = root_seg + '//' + name + ".png"
        os.remove(mask_path)
        os.remove(ssdna_path)


# -*- coding: utf-8 -*-
"""
i am so lost
"""

# Images_01_Starting_Code
# Engineer Your World

#HEY YO THIS IS PIERCE I DID NOT FORGET TO DO THIS
import cv2
import numpy
import os.path

def newTrackbarPos(slider_name, window_name, min_val, max_val):
    current_pos = cv2.getTrackbarPos(slider_name, window_name)
    if current_pos < min_val:
        print("hey that's too low")
        cv2.setTrackbarPos(slider_name, window_name, min_val)
        return min_val
    elif current_pos > max_val:
        print("hey that's too high")
        cv2.setTrackbarPos(slider_name, window_name, max_val)
        return max_val
    else:
        return current_pos

print ("Save your original image in the same folder as this program.")
filename_valid = False
while filename_valid == False:
    filename = input("Enter the name of your file, including the "\
                                 "extension, and then press 'enter': ")
    if os.path.isfile(filename) == True:
        filename_valid = True
    else:
        print ("Something was wrong with that filename. Please try again.")

original_image = cv2.imread(filename,1)
grayscale_image_simple = cv2.imread(filename, 0)
grayscale_image = cv2.cvtColor(grayscale_image_simple, cv2.COLOR_GRAY2BGR)

cv2.namedWindow('Original Image')
cv2.namedWindow('Grayscale Image')
#cv2.namedWindow('Red Parts of Image')
#cv2.namedWindow('Yellow Parts of Image')
cv2.namedWindow('Customized Image')
cv2.namedWindow('Slider')

image_height = original_image.shape[0]
image_width = original_image.shape[1]
image_channels = original_image.shape[2]

color1_paper = numpy.zeros((image_height,image_width,image_channels), numpy.uint8)
color2_paper = numpy.zeros((image_height,image_width,image_channels),
                           numpy.uint8)
color3_paper = numpy.zeros((image_height,image_width,image_channels),
                           numpy.uint8)
color4_paper = numpy.zeros((image_height,image_width,image_channels),
                           numpy.uint8)
color5_paper = numpy.zeros((image_height,image_width,image_channels), numpy.uint8)
color6_paper = numpy.zeros((image_height,image_width,image_channels), numpy.uint8)
color7_paper = numpy.zeros((image_height, image_width, image_channels), numpy.uint8)

color1_paper[0:image_height,0:image_width, 0:image_channels] = [255,153,204]
color2_paper[0:image_height,0:image_width, 0:image_channels] = [51,102,255]
color3_paper[0:image_height,0:image_width, 0:image_channels] = [255,204,0]
color4_paper[0:image_height,0:image_width, 0:image_channels] = [226,29,139]
color5_paper[0:image_height,0:image_width, 0:image_channels] = [204,255,255]
color6_paper[0:image_height,0:image_width, 0:image_channels] = [255,255,153]
color7_paper[0:image_height,0:image_width, 0:image_channels] = [153,153,255]
cv2.createTrackbar('grayscaletrack', 'Slider', 36, 254, lambda x:None)
cv2.createTrackbar('grayscaletrack2', 'Slider', 73, 254, lambda x:None)
cv2.createTrackbar('grayscaletrack3', 'Slider', 110, 254, lambda x:None)
cv2.createTrackbar('grayscaletrack4', 'Slider', 142, 254, lambda x:None)
cv2.createTrackbar('grayscaletrack5', 'Slider', 178, 254, lambda x:None)
cv2.createTrackbar('grayscaletrack6', 'Slider', 218, 254, lambda x:None)

cv2.imshow('Original Image', original_image)
cv2.imshow('Grayscale Image',grayscale_image)
#'move gray and original images here'

#color1 is red color2 is yellow color3 is green color4 is pink-purply
#the comment above can shut up because it is wrong

keypressed = 1
while(keypressed != 27 and keypressed != ord('s')): #add min and max
    grayscale_break = newTrackbarPos('grayscaletrack', 'Slider', 0, 36) #color 1 and 2
    grayscale_break2 = newTrackbarPos('grayscaletrack2', 'Slider', 37, 73) #mixed color above and 3
    grayscale_break3 = newTrackbarPos('grayscaletrack3', 'Slider', 74, 110) #mixed color above and 4
    grayscale_break4 = newTrackbarPos('grayscaletrack4', 'Slider', 111, 142) #mixed color above and 5
    grayscale_break5 = newTrackbarPos('grayscaletrack5', 'Slider', 143, 178) #mixed color above and 6
    grayscale_break6 = newTrackbarPos('grayscaletrack6', 'Slider', 179, 218) #mixed color above and 7

    #below is the ugly code that goes into the sliders
    min_grayscale_for_color1 = [0,0,0]
    max_grayscale_for_color1 = [grayscale_break,grayscale_break,grayscale_break]
    min_grayscale_for_color2 = [grayscale_break+1,grayscale_break+1,
                                grayscale_break+1]

    max_grayscale_for_color2 = [grayscale_break2,grayscale_break2,grayscale_break2]
    min_grayscale_for_color3 = [grayscale_break2+1,grayscale_break2+1,
                                grayscale_break2+1]

    max_grayscale_for_color3 = [grayscale_break3,grayscale_break3,grayscale_break3]
    min_grayscale_for_color4 = [grayscale_break3+1,grayscale_break3+1,
                                grayscale_break3+1]

    max_grayscale_for_color4 = [grayscale_break4,grayscale_break4,grayscale_break4]
    min_grayscale_for_color5 = [grayscale_break4+1, grayscale_break4+1, grayscale_break4+1]

    max_grayscale_for_color5 = [grayscale_break5,grayscale_break5,grayscale_break5]
    min_grayscale_for_color6 = [grayscale_break5+1,grayscale_break5+1,
                                grayscale_break5+1]

    max_grayscale_for_color6 = [grayscale_break6,grayscale_break6,grayscale_break6]
    min_grayscale_for_color7 = [grayscale_break6+1,grayscale_break6+1,
                                grayscale_break6+1]

    max_grayscale_for_color7 = [255,255,255]

    min_grayscale_for_color1 = numpy.array(min_grayscale_for_color1, dtype = "uint8")
    max_grayscale_for_color1 = numpy.array(max_grayscale_for_color1, dtype = "uint8")
    min_grayscale_for_color2 = numpy.array(min_grayscale_for_color2,
                                           dtype = "uint8")
    max_grayscale_for_color2 = numpy.array(max_grayscale_for_color2,
                                           dtype = "uint8")
    min_grayscale_for_color3 = numpy.array(min_grayscale_for_color3, dtype = "uint8")
    max_grayscale_for_color3 = numpy.array(max_grayscale_for_color3, dtype = "uint8")
    min_grayscale_for_color4 = numpy.array(min_grayscale_for_color4, dtype = "uint8")
    max_grayscale_for_color4 = numpy.array(max_grayscale_for_color4, dtype = "uint8")
    min_grayscale_for_color5 = numpy.array(min_grayscale_for_color5, dtype = "uint8")
    max_grayscale_for_color5 = numpy.array(max_grayscale_for_color5, dtype = "uint8")
    min_grayscale_for_color6 = numpy.array(min_grayscale_for_color6, dtype = "uint8")
    max_grayscale_for_color6 = numpy.array(max_grayscale_for_color6, dtype = "uint8")
    min_grayscale_for_color7 = numpy.array(min_grayscale_for_color7, dtype = "uint8")
    max_grayscale_for_color7 = numpy.array(max_grayscale_for_color7, dtype = "uint8")

    block_all_but_the_color1_parts = cv2.inRange(grayscale_image,
                                              min_grayscale_for_color1,
                                              max_grayscale_for_color1)
    block_all_but_the_color2_parts = cv2.inRange(grayscale_image,
                                                 min_grayscale_for_color2,
                                                 max_grayscale_for_color2)
    block_all_but_the_color3_parts = cv2.inRange(grayscale_image,
                                                 min_grayscale_for_color3,
                                                 max_grayscale_for_color3)
    block_all_but_the_color4_parts = cv2.inRange(grayscale_image,
                                                 min_grayscale_for_color4,
                                                 max_grayscale_for_color4)
    block_all_but_the_color5_parts = cv2.inRange(grayscale_image,
                                                 min_grayscale_for_color5,
                                                 max_grayscale_for_color5)
    block_all_but_the_color6_parts = cv2.inRange(grayscale_image,
                                                 min_grayscale_for_color6,
                                                 max_grayscale_for_color6)
    block_all_but_the_color7_parts = cv2.inRange(grayscale_image,
                                                 min_grayscale_for_color7,
                                                 max_grayscale_for_color7)

    color1_parts_of_image = cv2.bitwise_or(color1_paper, color1_paper,
                                        mask = block_all_but_the_color1_parts)
    color2_parts_of_image = cv2.bitwise_or(color2_paper, color2_paper,
                                           mask = block_all_but_the_color2_parts)
    color3_parts_of_image = cv2.bitwise_or(color3_paper, color3_paper,
                                           mask = block_all_but_the_color3_parts)
    color4_parts_of_image = cv2.bitwise_or(color4_paper, color4_paper,
                                           mask = block_all_but_the_color4_parts)
    color5_parts_of_image = cv2.bitwise_or(color5_paper, color5_paper,
                                           mask = block_all_but_the_color5_parts)
    color6_parts_of_image = cv2.bitwise_or(color6_paper, color6_paper,
                                           mask = block_all_but_the_color6_parts)
    color7_parts_of_image = cv2.bitwise_or(color7_paper, color7_paper,
                                           mask = block_all_but_the_color7_parts)

    customized_image = cv2.bitwise_or(color1_parts_of_image, color2_parts_of_image)
    customized_image = cv2.bitwise_or(customized_image, color3_parts_of_image)
    customized_image = cv2.bitwise_or(customized_image, color4_parts_of_image)
    customized_image = cv2.bitwise_or(customized_image, color5_parts_of_image)
    customized_image = cv2.bitwise_or(customized_image, color6_parts_of_image)
    customized_image = cv2.bitwise_or(customized_image, color7_parts_of_image)

    #cv2.imshow('Red Parts of Image',red_parts_of_image)
    #cv2.imshow('Yellow Parts of Image',yellow_parts_of_image)
    cv2.imshow('Customized Image',customized_image)

    keypressed = cv2.waitKey(1)
if keypressed == 27:
    cv2.destroyAllWindows()
elif keypressed == ord('s'):
     filename_valid = False
     while filename_valid == False:
        new_filename = input('Enter the new name of your custom image; be sure to include the file type as well-')
     else:
        print('That file name is not valid')
        #cv2.imwrite('photo_GS_1.jpg',grayscale_image)
        cv2.imwrite(new_filename+'.jpg',customized_image)
        cv2.destroyAllWindows()


from django.urls import path

from . import views

#app_name = 'polls'
urlpatterns = [
    path("test/", views.test),
    path("get/", views.get_all_users, name="get_all_users"),
    path("add/", views.add_users, name="add_users"),
    path("get/<int:pk>/", views.get_user_by_id, name="Get_User_By_Id"),
    path("update/<int:pk>/", views.update_user, name="Update_User"),
    path("delete/<int:pk>/", views.delete_user, name="Delete_User"),
]


from multiprocessing import Process
import feed, algo
import glob, time, copy

waitTime = 1800

def buildList():

	print("Built the list.")

	returnList = []
	with open("stocks", "r") as f:
		for i in f:
			returnList.append(i.rstrip())
	return returnList

def main():

	print("Starting feed programs")

	stocks = copy.copy(buildList())
	print(stocks)

	for stock in stocks:
		print(stock)
		client = stocks.index(stock) + 10
		print(client)
		print("Starting feed for {}".format(stock))
		p = Process(target=feed.main, args=(stock, client))
		p.start()
		print("Feed started. Client {}".format(client))

	print("Feed programs started")

if __name__ == '__main__':

	main()


import requests
from bs4 import BeautifulSoup

def get_mps_links(url):
    mps_urls = []
    page = requests.get(url)
    soup = BeautifulSoup(page.content, 'html5lib')
    link_tags = soup.select('.fraction_session_element .col-md-10 h5 a')
    for link_tag in link_tags:
        mps_urls.append(link_tag['href'])
    return mps_urls


from numpy import linspace
from ejercicio1 import intenumcomp
import matplotlib.pyplot as plt
import numpy as np
"""
la cantidad de n intervalos debe ser escogida de forma que la long
del intervalo sea <= a 0,1
"""
def senint(x):
    y = []
    for i in x:
        n = int(np.ceil(10*i))
        y.append(intenumcomp(np.cos,0,i,n,"trapecio"))
    return y

x_plot = np.arange(0,2*np.pi,0.5)
y_senint = senint(x_plot)
sen_plot = np.sin(x_plot)

plt.plot(x_plot,y_senint,label="Senint(x)")
plt.plot(x_plot,sen_plot,label="sen(x)")
plt.grid()
plt.legend()
plt.show()


# -*- coding: utf-8 -*-
"""
Created on Wed Jul 14 22:35:53 2021

@author: prabal
"""

import cv2
import numpy as np
import math
from vanishing_point_sp import *
from homography_projection import *
import glob
from utils_2 import get_imgL

def position_framework(c0,c1,c4,cTV,dict_m,dict_s):
    color = (255,255,255);thickness = 1
    img_G = cTV.img.copy()
    line_xb = [(0,0),(1600,0)];line_yb = [(1600,0),(1600,900)]
    frame_array = []
    for ii in range(len(dict_s['c0'])):
        m0 = cv2.imread(dict_m['c0'][ii]);im0 = cv2.imread(dict_s['c0'][ii])
        m0 = cv2.resize(m0,(1600,900),fx=0,fy=0, interpolation = cv2.INTER_CUBIC);im0 = cv2.resize(im0,(1600,900),fx=0,fy=0, interpolation = cv2.INTER_CUBIC)
        c0.img = im0;c0.mask = m0
        m1 = cv2.imread(dict_m['c1'][ii]);im1 = cv2.imread(dict_s['c1'][ii])
        m1 = cv2.resize(m1,(1600,900),fx=0,fy=0, interpolation = cv2.INTER_CUBIC);im1 = cv2.resize(im1,(1600,900),fx=0,fy=0, interpolation = cv2.INTER_CUBIC)
        c1.img = im1;c1.mask = m1
#        m2 = cv2.imread(dict_m['c2'][ii]);im2 = cv2.imread(dict_s['c2'][ii])
#        m2 = cv2.resize(m2,(1600,900),fx=0,fy=0, interpolation = cv2.INTER_CUBIC);im2 = cv2.resize(im2,(1600,900),fx=0,fy=0, interpolation = cv2.INTER_CUBIC)
#        c2.img = im2;c2.mask = m2
#        m3 = cv2.imread(dict_m['c3'][ii]);im3 = cv2.imread(dict_s['c3'][ii])
#        m3 = cv2.resize(m3,(1600,900),fx=0,fy=0, interpolation = cv2.INTER_CUBIC);im3 = cv2.resize(im3,(1600,900),fx=0,fy=0, interpolation = cv2.INTER_CUBIC)
#        c3.img = im3;c3.mask = m3
        m4 = cv2.imread(dict_m['c4'][ii]);im4 = cv2.imread(dict_s['c4'][ii])
        m4 = cv2.resize(m4,(1600,900),fx=0,fy=0, interpolation = cv2.INTER_CUBIC);im1 = cv2.resize(im4,(1600,900),fx=0,fy=0, interpolation = cv2.INTER_CUBIC)
        c4.img = im4;c4.mask = m4
        cTV.img = img_G.copy()
        imgDisp,cntr = locVehicle(c0,c1,c4,cTV,ret_type='img')
        masks = dict_m['c4'][0:ii+1];imgs = dict_s['c4'][0:ii+1]
        if len(masks) >1:
            poc = optFlowVector_PoC(masks,imgs,horizon = True)
            #poc = optFlowVector_PoC(masks,imgs)
            #poc = optFlowVector_PoC(masks,imgs,concensus = 'rad')
            #poc = optFlowVector_PoC(masks,imgs,concensus = 'rad',horizon = True)
            cntr_h = frameChange(cntr,cTV,c4)
            line_c4 = [tuple(cntr_h),tuple(poc)]
            if intersection_mc([line_xb,line_c4])[0]<= 1600:
                poc_ = intersection_mc([line_xb,line_c4])
            else:
                poc_ = intersection_mc([line_yb,line_c4])
            poc_w = frameChange(poc_,c4,cTV)
            imgDisp = cv2.line(imgDisp,tuple(cntr),tuple(poc_w),color,thickness)
            imgDisp,vpcx_ = drawVPLinePose(imgDisp,c4,cntr,line_xb,line_yb)
            m1,_ = mc([tuple(cntr),tuple(poc_w)]);m2,_=mc([tuple(cntr),tuple(vpcx_)])
            m1,m2 = abs(m1),abs(m2)
            dev_angle = math.atan(abs((m1-m2)/(1+m1*m2)))*180/np.pi
            text = 'deviation from VP: ' + str(dev_angle)
            imgDisp = cv2.putText(imgDisp,text, (1100, 100),cv2.FONT_HERSHEY_SIMPLEX,1,
                 (0, 0, 255),2, cv2.LINE_AA, False)
            print(dev_angle)
        cv2.imshow("Pose", imgDisp)
        cv2.waitKey(300)
        frame_array.append(imgDisp)

        if cv2.waitKey(10) & 0xFF == ord('q') :
            break
    cv2.destroyAllWindows()
    return frame_array

def saveframe2vid(frame_array,pathOut,fps):
    out = cv2.VideoWriter(pathOut,cv2.VideoWriter_fourcc(*'DIVX'), fps, (1600, 900))
    for i in range(len(frame_array)):
        # writing to a image array
        out.write(frame_array[i])
    out.release()

def drawVPLinePose(img_,cx,cntr,line_xb,line_yb):
    color = (0,255,0);thickness=2
    vpcx = ShowVanishingPoint(cx.img.copy(),getRho_Theta(cx.img.copy(),cx.x_lines))
    line_cx = [tuple(cntr),tuple(vpcx)]
    if intersection_mc([line_xb,line_cx])[0]<= 1600:
        vpcx_ = intersection_mc([line_xb,line_cx])
    else:
        vpcx_ = intersection_mc([line_yb,line_cx])
    vpcx_ = frameChange(vpcx_,cx,cTV)
    img_ = drawDottedline(img_,tuple(cntr),tuple(vpcx_),color,thickness)
    return img_,vpcx_

def frameChange(pt,cx,cy):  #locating pt in cy with originally being in cx
    hcx = cx.homography(cy)
    pt_temp = getH_InvCoordinate(pt)
    pt_ = np.matmul(hcx,np.transpose(pt_temp))
    pt__ = getH_InvCoordinate(pt_,inv=True)
    return pt__

def optFlowVector_PoC(masks,imgs,concensus = 'grad',horizon = False):   #concensus = 'grad/rad'
    # Parameters for ShiTomasi corner detection
#    feature_params = dict(maxCorners=100, qualityLevel=0.4, minDistance=2, blockSize=7)
#
#    # Parameters for Lucas Kanade optical flow
#    lk_params = dict(
#        winSize=(15, 15),
#        maxLevel=2,
#        criteria=(cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03),
#    )
    feature_params = dict( maxCorners = 200,
                       qualityLevel = 0.2,
                       minDistance = 2,
                       blockSize = 7 )
    lk_params = dict(winSize = (15,15), maxLevel = 3, criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03))
    # Create random colors
    color = np.random.randint(0, 255, (1000, 3),dtype = np.int)
    #color = (0, 255, 0)
    first_frame = cv2.imread(masks[0])
    first_frame = cv2.resize(first_frame,(1600,900),fx=0,fy=0, interpolation = cv2.INTER_CUBIC)
    prev_gray = cv2.cvtColor(first_frame, cv2.COLOR_BGR2GRAY)
    prev = cv2.goodFeaturesToTrack(prev_gray, mask = None, **feature_params)
    mask = np.zeros_like(first_frame)
    points = np.array([]);q = np.array([False])
    for j in range(len(masks)):
        m = cv2.imread(masks[j]);im = cv2.imread(imgs[j])
        m = cv2.resize(m,(1600,900),fx=0,fy=0, interpolation = cv2.INTER_CUBIC)
        im = cv2.resize(im,(1600,900),fx=0,fy=0, interpolation = cv2.INTER_CUBIC)
        gray = cv2.cvtColor(m, cv2.COLOR_BGR2GRAY)
        new, status, error = cv2.calcOpticalFlowPyrLK(prev_gray, gray, prev, None, **lk_params)
        good_old = prev[status == 1]
        good_new = new[status == 1]
        of_vect = []
        if horizon:
            of_vect = [[(1740, -16), (141, -36)]]
#        horizon = [(1740, -16), (141, -36)]

        for i, (new, old) in enumerate(zip(good_new, good_old)):
            # Returns a contiguous flattened array as (x, y) coordinates for new point
            a, b = new.ravel()
            # Returns a contiguous flattened array as (x, y) coordinates for old point
            c, d = old.ravel()
            col_ = (int(color[i][0]),int(color[i][1]), int (color[i][2]))
            if (c-a) !=0:
                #print(str(i),(b-d)/(c-a))
                if concensus == 'grad':
                    grad = (b-d)/(c-a)
                    points = np.append(points,grad)
                elif concensus == 'rad':
                    rad = math.sqrt((a-b)**2+(c-d)**2)
                    points = np.append(points,rad)
                if len(points)>2:
                    q = is_outlier(points)
            if q[-1].item() is False:
                #print('(a,b)',(a,b),'(c,d)',(c,d))
                # Draws line between new and old position with green color and 2 thickness
                mask = cv2.line(mask, (a, b), (c, d), tuple(col_), 2)
                # Draws filled circle (thickness of -1) at new position with green color and radius of 3
                m = cv2.circle(m, (a, b), 3, tuple(col_), -1)
                of_vect.append([(c,d),(a,b)])
#                of_vect = [(c,d),(a,b)]
#                line__ = [of_vect,horizon]
#                if j>0:
#                    poc = intersection_mc(line__)
#                    print(poc)
        if j>0:
            poc = intersection_mc(of_vect)
        else:
            poc = None
            #print(poc)
        # Overlays the optical flow tracks on the original frame
        output = cv2.add(im, mask)
        # Updates previous frame
        prev_gray = gray.copy()
        # Updates previous good feature points
        prev = good_new.reshape(-1, 1, 2)
        # Opens a new window and displays the output frame
#        cv2.imshow("sparse optical flow", output)
#        cv2.waitKey(300)
#        if cv2.waitKey(10) & 0xFF == ord('q') :
#            break
#    cv2.destroyAllWindows()
    print(len(points))
    print(len(q))
    points = points[np.where(q==False)]
    print('Outliers',sum(q))
#    print(of_vect)
    return poc

def is_outlier(points, thresh=0.25):
    """
    Returns a boolean array with True if points are outliers and False
    otherwise.

    Parameters:
    -----------
        points : An numobservations by numdimensions array of observations
        thresh : The modified z-score to use as a threshold. Observations with
            a modified z-score (based on the median absolute deviation) greater
            than this value will be classified as outliers.

    Returns:
    --------
        mask : A numobservations-length boolean array.

    References:
    ----------
        Boris Iglewicz and David Hoaglin (1993), "Volume 16: How to Detect and
        Handle Outliers", The ASQC Basic References in Quality Control:
        Statistical Techniques, Edward F. Mykytka, Ph.D., Editor.
    """
    if len(points.shape) == 1:
        points = points[:,None]
    median = np.median(points, axis=0)
    diff = np.sum((points - median)**2, axis=-1)
    diff = np.sqrt(diff)
    med_abs_deviation = np.median(diff)

    modified_z_score = 0.6745 * diff / med_abs_deviation

    return modified_z_score > thresh

if __name__ =="__main__":
    ptsc4=np.float32([[307,473],[218,304],[1341,120],[1359,298],[None,None],[1071,213],[1151,108],[1167,180],[431,432],[805,180],[537,237],[988,608],[1517,169],[None,None],[843,290]]) #c4
    ptsTV=np.float32([[824, 276],[871 ,276],[824 ,559],[780 ,394],[780,276],[824,401],[868,528],[824,441],[824 ,286],[870,376],[870,317],[None,None],[780,523],[824,243],[824,339],[700,464],[691,323],[730,322],[570,335]]) #TV
    ptsc1=np.float32([[1043,196],[1301,285],[None,None],[266,291],[880,136],[303,472],[None,None],[None,None],[1010,209],[762,654],[1150,389],[759,168],[None,None],[1150,157],[769,302]]) #c1
    ptsc0=np.float32([[1486,509],[None,None],[None,None],[649,469],[1283,449],[510,533],[None,None],[262,536],[1393,512],[467,703],[1345,712],[1090,454],[124,486],[None,None],[928,525],[529,421],[1012,397],[1033,414],[907,363]]) #c0
    ptsc3=np.float32([[957,399],[662,630],[171,143],[604,194],[1082,301],[420,226],[None,None],[335,197],[883,376],[187,310],[382,436],[913,261],[None, None],[1300,510],[597,284]]) #c3
    x_linesC0 = [[(649,469),(1283,449)],[(1486,509),(262,536)],[(467,703),(1345,712)]]
    x_linesC1 = [[(266,291),(880,136)],[(303,472),(1043,196)],[(762,654),(1301,285)]]
    x_linesC4 = [[(1517,169),(988,608)],[(307,473),(1341,120)],[(218,304),(1151,108)]]
    x_linesC3 = [[(604,194),(1082,301)],[(957,399),(171,143)],[(662,630),(187,310)]]

    y_linesC0 = [[(1486,509), (1283,449)]]
    y_linesC1 = [[(1301,285), (880,136)]]
    y_linesC4 = [[(301, 479), (244, 295)]]
    y_linesC3 = [[(662,630), (1082,301)]]
    y_linesC2 = [[(702,178), (282,359)]]
    y_lines_cTV4 = [[(824,339),(871,339)],[(780,523),(871,523)]]

    imgTV = cv2.imread('google_top_view_1600x900.jpg')
    imgc0 = cv2.imread('frame_0.jpg')
    imgc1 = cv2.imread('frame_1.jpg')
    imgc2 = cv2.imread('frame_2.jpg')
    imgc3 = cv2.imread('frame_3.jpg')
    imgc4 = cv2.imread('frame_4.jpg')
    add_lm_c0 = get_imgL(cam='c0',img_type = 'Mask')
    add_ls_c0 = get_imgL(cam='c0',img_type='Screenshots')
    add_lm_c1 = get_imgL(cam='c1',img_type = 'Mask')
    add_ls_c1 = get_imgL(cam='c1',img_type='Screenshots')
    add_lm_c2 = get_imgL(cam='c2',img_type = 'Mask')
    add_ls_c2 = get_imgL(cam='c2',img_type='Screenshots')
    add_lm_c3 = get_imgL(cam='c3',img_type = 'Mask')
    add_ls_c3 = get_imgL(cam='c3',img_type='Screenshots')
    add_lm_c4 = get_imgL(cam='c4',img_type = 'Mask')
    add_ls_c4 = get_imgL(cam='c4',img_type='Screenshots')

    dict_m = {'c0':add_lm_c0,'c1':add_lm_c1,'c2':add_lm_c2,'c3':add_lm_c3,'c4':add_lm_c4}
    dict_s = {'c0':add_ls_c0,'c1':add_ls_c1,'c2':add_ls_c2,'c3':add_ls_c3,'c4':add_ls_c4}

    y_lines_cTV0_3 = [[(780,394),(871,394)],[(824,339),(871,339)]]
    c0 = camera_projection('c0',imgc0,None,ptsc0,x_linesC0,y_linesC0 )
    c1 = camera_projection('c1',imgc1,None,ptsc1,x_linesC1,y_linesC1)
    c4 = camera_projection('c4',imgc4,None,ptsc4,x_linesC4,y_linesC4)
    #c3 = camera_projection('c3',img3,img3_mask,ptsc3,x_linesC3,y_linesC3)
    cTV = camera_projection('cTV',imgTV,None,ptsTV,None,None)
    frame_array_loc = position_framework(c0,c1,c4,cTV,dict_m,dict_s)


from django.conf.urls import url,include
from . import views
from django.conf import settings
from django.conf.urls.static import static

urlpatterns=[
    url('^$',views.index,name = 'index'),
    url(r'^search/', views.search_results, name='search_results'),
    url(r'^accounts/', include('registration.backends.simple.urls')),
    url(r'^profile/create/',views.create_profile,name = 'create_profileUrl'),
    url(r'^business/view',views.view_business,name = 'viewbusinessUrl'),
    url(r'^business/',views.business,name = 'business'),
    url(r'^new/post$', views.new_post, name='new_post'),
    url(r'^post/',views.post,name = 'post'),
    url(r'^leave/hood',views.leave_hood,name = 'leave_hoodUrl'),

]

if settings.DEBUG:
    urlpatterns+= static(settings.MEDIA_URL, document_root = settings.MEDIA_ROOT)


"""API Serializers & Views"""

from . import models
from django.utils import timezone
from rest_framework import serializers, viewsets
from rest_framework import permissions

# Serializers define the API representation.
class RawFlagSerializer(serializers.HyperlinkedModelSerializer):

    flagger = serializers.HiddenField(default=serializers.CurrentUserDefault())

    class Meta:
        model = models.RawFlag
        fields = [
            'id',
            'url',
            'flagger',
            'reported_user_id',
            'reported_tweet_id',
            'language',
        ]
        read_only_fields = [
            'created',
            'modified'
        ]

# ViewSets define the view behavior.
class RawFlagViewSet(viewsets.ModelViewSet):
    queryset = models.RawFlag.objects.all()
    serializer_class = RawFlagSerializer

    def get_permissions(self):
        if self.action == 'create':
            permission_classes = [permissions.IsAuthenticated]
        else:
            permission_classes = [permissions.IsAdminUser]
        return [permission() for permission in permission_classes]


#!/usr/bin/env python
# coding: utf-8

import argparse
import collections
from xml.etree import ElementTree as ET
import xml.dom.minidom
import json
import os
import os.path as fs
import logging
import tempfile
import shutil
import hashlib
import sys

from .common.chunks import ModType
from .common.chunks import mod_types_to_str
from .common.extract_utils import extract_submission
from .common.source_doc import load_sources_docs
from .common import src_mapping
from .common.submissions import run_over_submissions
from .common.text_proc import seg_text
from . import common_runner

class SrcRetrievalMetaGenerator:
    def __init__(self, opts):
        self._opts = opts
        if opts.mapping is not None:
            self._mapping = src_mapping.SrcMap()
            self._mapping.from_csv(opts.mapping)
        else:
            ids = _load_ids(opts.ids_file)
            self._mapping = create_mapping(opts.subm_dir,
                                           use_filename_as_id = opts.use_filename_as_id,
                                           ids = ids)

        self._src_map = None
        self._init_src_map()

        self._out_dir = fs.join(opts.src_retr_out_dir, 'meta')
        if not fs.exists(self._out_dir):
            os.makedirs(self._out_dir)

    def _init_src_map(self):
        self._src_map = collections.defaultdict(lambda: 0)

    def __call__(self, susp_doc, chunk, offsets):
        if not offsets:
            return

        susp_id = susp_doc.get_susp_id()
        src_id = self._mapping.get_src_by_filename(
            susp_id,
            chunk.get_orig_doc_filename()).get_ext_id()

        self._src_map[src_id] += len(chunk.get_mod_sents())

    def on_susp_end(self, susp_doc, _):
        #TODO lang
        #TODO version
        susp_id = susp_doc.get_susp_id()
        meta = {"language": "ru",
                "plagiarism": [],
                "suspicious-document": "%s.txt" % susp_id}

        for src_id in self._src_map:
            if self._src_map[src_id] < self._opts.min_sent_cnt:
                continue
            meta["plagiarism"].append({
                "reused_sent_cnt": self._src_map[src_id],
                "id": src_id
            })

        self._init_src_map()

        out_path = fs.join(
            self._out_dir,
            "%s.json" % susp_id)
        with open(out_path, 'w') as out:
            json.dump(meta, out,
                      indent=4, separators=(',', ': '))

    def get_name(self):
        return 'SrcRetrievalMetaGenerator'

class SimilarDocumentsMetaGenerator(SrcRetrievalMetaGenerator):
    def __init__(self, opts):
        super().__init__(opts)

        self._out_path = fs.join(
            self._opts.src_retr_out_dir, "retrieval_data.csv")

        with open(self._out_path, 'w') as outf:
            outf.write("srcID,srcTitle,dstID,dstTitle,rank,reused_sent_cnt\n")

    def on_susp_end(self, susp_doc, _):
        susp_id = susp_doc.get_susp_id()

        items = list(self._src_map.items())
        items.sort(key = lambda t : -t[1])

        with open(self._out_path, 'a') as outf:
            for num, (src_id, reused_sent_cnt) in enumerate(items):
                if reused_sent_cnt < self._opts.min_sent_cnt:
                    continue
                outf.write("%s,,%s,,%d,%d\n" % (susp_id, src_id, num + 1, reused_sent_cnt))

        self._init_src_map()

    def get_name(self):
        return 'SimilarDocumentsMetaGenerator'

class TextAlignmentTaskGenerator(object):
    def __init__(self, opts):
        self._opts = opts
        if opts.mapping is not None:
            self._mapping = src_mapping.SrcMap()
            self._mapping.from_csv(opts.mapping)
        else:
            self._mapping = create_mapping(opts.subm_dir,
                                           use_filename_as_id = opts.use_filename_as_id)

        self._xml_map = {}
        self._pairs = []

        susp_dir = fs.join(opts.text_align_out_dir, 'susp')
        src_dir = fs.join(opts.text_align_out_dir, 'src')
        meta_dir = fs.join(opts.text_align_out_dir, 'meta')
        for d in [susp_dir, src_dir, meta_dir]:
            if not fs.exists(d):
                os.makedirs(d)

        if fs.exists(fs.join(meta_dir, "pairs")):
            os.remove(fs.join(meta_dir, "pairs"))

    def __call__(self, susp_doc, chunk, offsets):
        if not offsets:
            return

        susp_id = susp_doc.get_susp_id()
        src_id = self._mapping.get_src_by_filename(
            susp_id,
            chunk.get_orig_doc_filename()).get_ext_id()

        susp_filename = susp_id + ".txt"
        src_filename = src_id + ".txt"
        if src_id not in self._xml_map:
            #TODO add version
            self._xml_map[src_id] = ET.Element('document', reference=susp_filename)
            self._pairs.append((susp_filename, src_filename))

        for t in offsets:
            if t is None:
                continue
            src_offs_beg, src_offs_end, err = t
            e = ET.Element('feature',
                           name='plagiarism',
                           this_offset=str(susp_doc.get_offs_for_text(chunk.get_chunk_id())),
                           this_length=str(len(chunk.get_mod_text())),
                           source_reference=src_filename,
                           source_offset=str(src_offs_beg),
                           source_length=str(src_offs_end - src_offs_beg),
                           type="manual",
                           obfuscation=mod_types_to_str(chunk.get_all_mod_types()),
                           #TODO
                           this_language="ru",
                           source_language="ru",
                           txt_extraction_err=str(err))
            self._xml_map[src_id].append(e)

    def _get_as_xml(self, src_id):
        return self._xml_map[src_id]

    def _finalize_pair(self, susp_id, src_id):
        meta_xml = self._get_as_xml(src_id)
        ugly_xml_str = ET.tostring(meta_xml, 'utf-8')

        ugly_xml = xml.dom.minidom.parseString(ugly_xml_str)

        pretty_xml = ugly_xml.toprettyxml()

        out_path = fs.join(
            self._opts.text_align_out_dir,
            "meta",
            "suspicious-document%s-source-document%s.xml" % (susp_id, src_id))
        with open(out_path, 'w') as out:
            out.write(pretty_xml)

    def on_susp_end(self, susp_doc, sources):
        #TODO
        #self._join_adjacent_chunks()

        susp_id = susp_doc.get_susp_id()
        #write susp text
        susp_file = fs.join(self._opts.text_align_out_dir, 'susp', "%s.txt" % susp_id)
        with open(susp_file, 'w') as f:
            susp_doc.write_susp_doc(f)

        #write sources
        src_dir = fs.join(self._opts.text_align_out_dir, 'src')
        write_sources_to_files(self._mapping, susp_id, sources, src_dir,
                               ext_id_as_filename=True)

        for src_id in self._xml_map:
            self._finalize_pair(susp_id, src_id)
        self._xml_map = {}
        #update pairs
        with open(fs.join(self._opts.text_align_out_dir, "meta", "pairs"), 'a') as f:
            for pair in self._pairs:
                f.write("%s %s\n" % pair)
        self._pairs = []

    def get_name(self):
        return 'TextAligmentMetaGenerator'

class SentRetrievalTaskGenerator:
    def __init__(self, opts):
        self._opts = opts
        self._src_sents = []
        self._tgt_sents = []
        self._seen_src_sents = set()
        self._seen_target_sents = set()
        self._pairs = []

    def _make_src_sent_id(self, prefix, susp_doc, chunk, num):
        return f'{prefix}_{susp_doc.get_susp_id()}_{chunk.get_id()}_{num}'

    def _make_sent_hash(self, sent_text):
        sha1 = hashlib.sha1()
        sha1.update(sent_text.encode('utf8'))
        return sha1.hexdigest()

    def _clean_sent(self, text):
        return text.strip().replace('\t', ' ').replace('\n', ' ')

    def _susps_sents(self, chunk):
        if chunk.has_mod_type(ModType.SSP) or chunk.has_mod_type(ModType.SEP):
            for s, _ in seg_text(chunk.get_mod_text()):
                yield s
        else:
            yield from chunk.get_mod_sents()

    def __call__(self, susp_doc, chunk, offsets):
        src_ids = []

        # print('chunk with id ', chunk.get_id(), 'num mods', len(chunk.get_mod_sents()), ' num origs', len(chunk.get_orig_sents()) )
        for num, s in enumerate(self._susps_sents(chunk)):
            src_sent = self._clean_sent(s)
            src_hash = self._make_sent_hash(src_sent)
            if src_hash not in self._seen_src_sents:
                self._seen_src_sents.add(src_hash)
                src_id = self._make_src_sent_id('src', susp_doc, chunk, num)
                src_ids.append(src_id)
                self._src_sents.append((src_id, src_sent))

        tgt_ids = []
        for num, s in enumerate(chunk.get_orig_sents()):
            tgt_sent = self._clean_sent(s)
            tgt_id = self._make_sent_hash(tgt_sent)
            tgt_ids.append(tgt_id)
            if tgt_id not in self._seen_target_sents:
                self._seen_target_sents.add(tgt_id)
                self._tgt_sents.append((tgt_id, tgt_sent))

        for src_id in src_ids:
            for tgt_id in tgt_ids:
                self._pairs.append((src_id, tgt_id))

    def _add_sents_from_sources(self, sources):
        for source in sources.values():
            for s, _ in seg_text(source.get_text()):
                tgt_sent = self._clean_sent(s)
                tgt_id = self._make_sent_hash(tgt_sent)
                if tgt_id not in self._seen_target_sents:
                    self._seen_target_sents.add(tgt_id)
                    self._tgt_sents.append((tgt_id, tgt_sent))

    def on_susp_end(self, _, sources):

        self._add_sents_from_sources(sources)

        with open(f'{self._opts.out_prefix}.src', 'a') as f:
            for t in self._src_sents:
                f.write("%s\n" % '\t'.join(t))
        with open(f'{self._opts.out_prefix}.tgt', 'a') as f:
            for t in self._tgt_sents:
                f.write("%s\n" % '\t'.join(t))
        with open(f'{self._opts.out_prefix}.gold', 'a') as f:
            for t in self._pairs:
                f.write("%s\n" % '\t'.join(t))

        self._src_sents = []
        self._tgt_sents = []
        self._pairs = []

    def get_name(self):
        return 'SentRetrievalTaskGenerator'

class DumbDumper(object):
    def __init__(self, out_path = None):
        if out_path is None:
            self._out = sys.stdout
        else:
            self._out = open(out_path, 'w')

    def __call__(self, susp_doc, chunk, offsets):
        for t in offsets:
            if t is None:
                b,e,er = -1,-1,-1
            else:
                b,e,er = t
            self._out.write("%s,%s,%d,%d,%d\n" % (susp_doc.get_susp_id(),
                                                  chunk.get_chunk_id(), b,e,er))

    def on_susp_end(self, susp_doc, sources):
        pass

    def get_name(self):
        return 'DumbDumper'

class Generator(object):
    def __init__(self, opts, out_pipes):
        self._opts = opts
        self._out_pipes = out_pipes

    def process_chunk(self, susp_doc, chunk, sources):
        if chunk.get_mod_type() == ModType.ORIG or not chunk.get_orig_doc_filename():
            return
        source = sources[chunk.get_orig_doc_filename()]
        offsets = []
        for sent in chunk.get_orig_sents():
            offsets.append(source.get_sent_offs(sent))

        for pipe in self._out_pipes:
            try:
                pipe(susp_doc, chunk, offsets)
            except Exception as e:
                logging.warning("Id: %s - Failed to process offsets by %s: %s",
                                susp_doc.get_susp_id(), pipe.get_name(), e)

    def process_extracted_archive(self, susp_id, sources_dir, meta_file_path):
        chunks, chunks_errors = common_runner.create_chunks(susp_id, meta_file_path,
                                                            self._opts.version)

        if chunks_errors:
            logging.error("Id: %s - Errors while creating chunks:\n%s", susp_id,
                          "\n".join(str(e) for e in chunks_errors))
        sources = load_sources_docs(sources_dir)
        susp_doc = SuspDocGenerator(susp_id)
        susp_doc.add_chunks(chunks)
        for chunk in chunks:
            try:
                self.process_chunk(susp_doc, chunk, sources)
            except Exception as e:
                logging.warning("Id: %s - Failed to process chunk %s: %s",
                                susp_id, chunk.get_chunk_id(), e)

        for pipe in self._out_pipes:
            try:
                pipe.on_susp_end(susp_doc, sources)
            except Exception as e:
                logging.warning("Id: %s - Failed to finalize susp: %s",
                                susp_doc.get_susp_id(), e)

    def process_archive(self, archive_path, susp_id):

        temp_dir = tempfile.mkdtemp()
        try:
            sources_dir, meta_file_path = extract_submission(archive_path, temp_dir)
            self.process_extracted_archive(susp_id, sources_dir, meta_file_path)
        finally:
            shutil.rmtree(temp_dir)

    def process_submissions(self):
        ids = _load_ids(self._opts.ids_file)
        run_over_submissions(self._opts.subm_dir,
                             self.process_extracted_archive,
                             self._opts.limit_by_version,
                             include_ids_set = ids)

class SuspDocGenerator:
    def __init__(self, susp_id):
        super().__init__()
        self._susp_id = susp_id
        self._cur_offset = 0
        self._offsets_dict = {}
        self._text_parts = []

    def get_susp_id(self):
        return self._susp_id

    def add_text(self, text_id, text):

        suffix = '\n'
        self._text_parts.append(text)
        self._text_parts.append(suffix)
        text_offs = self._cur_offset
        self._offsets_dict[text_id] = text_offs
        self._cur_offset += len(text) + len(suffix)
        return text_offs

    def get_offs_for_text(self, text_id):
        return self._offsets_dict[text_id]

    def add_chunks(self, chunks):
        for chunk in chunks:
            self.add_text(chunk.get_chunk_id(),
                          chunk.get_mod_text())

    def write_susp_doc(self, out):
        for t in self._text_parts:
            out.write(t)

def write_sources_to_files(mapping, susp_id, sources, out_dir,
                           ext_id_as_filename = False):
    if not fs.exists(out_dir):
        os.makedirs(out_dir)

    for src_filename in sources:
        source_doc = sources[src_filename]
        src = mapping.get_src_by_filename(susp_id, src_filename)
        if ext_id_as_filename:
            filename = src.get_ext_id()
        else:
            filename = src.get_res_id()
        filepath = fs.join(
            out_dir, filename + ".txt")
        source_doc.write_text_to_file(filepath)

def create_mapping(subm_dir, limit_by_version = None,
                   use_filename_as_id = False, ids = None):
    mapping = src_mapping.SrcMap()

    def arc_proc(susp_id, sources_dir, _):
        src_mapping.add_src_from_dir(susp_id, sources_dir, mapping,
                                     use_filename_as_id)

    run_over_submissions(subm_dir, arc_proc, limit_by_version,
                         include_ids_set = ids)
    return mapping

#cli support
def _load_ids(ids_file):
    if ids_file:
        with open(ids_file, 'r') as f:
            return frozenset([int(l) for l in f])
    return None

def create_text_align_task(opts):
    pipes = [TextAlignmentTaskGenerator(opts)]
    gener = Generator(opts, pipes)
    gener.process_submissions()

def create_sent_retrieval_meta(opts):
    pipes = [SentRetrievalTaskGenerator(opts)]
    gener = Generator(opts, pipes)
    gener.process_submissions()

def create_src_retr_meta(opts):
    pipes = [SrcRetrievalMetaGenerator(opts)]
    gener = Generator(opts, pipes)
    gener.process_submissions()

def create_doc_sim_meta(opts):
    pipes = [SimilarDocumentsMetaGenerator(opts)]
    gener = Generator(opts, pipes)
    gener.process_submissions()

def create_pan_meta(opts):
    pipes = [SrcRetrievalMetaGenerator(opts),
             TextAlignmentTaskGenerator(opts)]
    gener = Generator(opts, pipes)
    gener.process_submissions()

def dumb_dump(opts):
    pipes = [DumbDumper(opts.out_file)]
    gener = Generator(opts, pipes)
    gener.process_submissions()

def gen_map(opts):
    ids = _load_ids(opts.ids_file)
    mapping = create_mapping(opts.subm_dir, opts.limit_by_version,
                             opts.use_filename_as_id, ids)
    with open(opts.mapping_file, 'w') as f:
        mapping.to_csv(f)

def create_sources(opts):
    ids = _load_ids(opts.ids_file)
    mapping = src_mapping.SrcMap()
    mapping.from_csv(opts.mapping)

    def arc_proc(susp_id, sources_dir, _):
        sources = load_sources_docs(sources_dir)
        write_sources_to_files(mapping, susp_id, sources, opts.out_dir,
                               opts.ext_id_as_filename)

    run_over_submissions(opts.subm_dir, arc_proc, opts.limit_by_version,
                         include_ids_set = ids)

def create_susp_docs(opts):
    def arc_proc(susp_id, _, meta_file_path):
        chunks, _ = common_runner.create_chunks(
            susp_id, meta_file_path, opts.version)
        out_path = fs.join(opts.out_dir, "%s.txt" % susp_id)
        with open(out_path, 'w') as f:
            susp_gen = SuspDocGenerator(susp_id)
            susp_gen.add_chunks(chunks)
            susp_gen.write_susp_doc(f)
    if not fs.exists(opts.out_dir):
        os.makedirs(opts.out_dir)

    ids = _load_ids(opts.ids_file)
    run_over_submissions(opts.subm_dir, arc_proc, opts.limit_by_version,
                         include_ids_set = ids)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--verbose", "-v", action="store_true", default = False)
    parser.add_argument("--version", "-V", default=None,
                        help="version of essay."
                        " If it is not specified, will be determined by essay id.")
    parser.add_argument("--limit_by_version", "-L", default=None,
                        help="process only essays with specified version."
                        "If not specified, process all found essays.")

    subparsers = parser.add_subparsers(help='sub-command help')

    dump_parser = subparsers.add_parser('dump',
                                        help='help of dump')

    dump_parser.add_argument("--subm_dir", "-i", required = True,
                             help = "directory with submissions")
    dump_parser.add_argument("--out_file", "-o", default=None)

    dump_parser.set_defaults(func = dumb_dump)

    gen_map_parser = subparsers.add_parser('gen_map',
                                           help='help of gen_map')

    gen_map_parser.add_argument("--subm_dir", "-i", required = True,
                                help = "directory with submissions")
    gen_map_parser.add_argument("--mapping_file", "-o", default = "src_mapping.csv",
                                help = "mapping file path")
    gen_map_parser.add_argument("--use_filename_as_id", "-u", action='store_true')
    gen_map_parser.add_argument("--ids_file", "-I", default='',
                                help = "use only those ids, otherwise process everything")

    gen_map_parser.set_defaults(func = gen_map)

    create_src_parser = subparsers.add_parser('create_src',
                                              help='help of create_src')

    create_src_parser.add_argument("--subm_dir", "-i", required = True,
                                   help = "directory with submissions")
    create_src_parser.add_argument("--mapping", "-m", default = "src_mapping.csv",
                                   help = "mapping file path")
    create_src_parser.add_argument("--out_dir", "-o", default="essay_src")
    create_src_parser.add_argument("--ext_id_as_filename", "-e", action="store_true")
    create_src_parser.add_argument("--ids_file", "-I", default='',
                                   help = "use only those ids, otherwise process everything")
    create_src_parser.set_defaults(func = create_sources)

    create_susp_parser = subparsers.add_parser('create_susp',
                                               help='help of create_susp')
    create_susp_parser.add_argument("--subm_dir", "-i", required = True,
                                    help = "directory with submissions")
    create_susp_parser.add_argument("--out_dir", "-o", default="essay_susp")
    create_susp_parser.add_argument("--ids_file", "-I", default='',
                                    help = "use only those ids, otherwise process everything")
    create_susp_parser.set_defaults(func = create_susp_docs)

    text_align_parser = subparsers.add_parser('text_align',
                                              help='help of text_align')
    text_align_parser.add_argument("--subm_dir", "-i", required = True,
                                   help = "directory with submissions")
    text_align_parser.add_argument("--text_align_out_dir", "-o",
                                   default="01-manual-plagiarism")
    text_align_parser.add_argument("--mapping", "-m", default = "src_mapping.csv",
                                   help = "mapping file path")
    text_align_parser.add_argument("--use_filename_as_id", "-u", action='store_true')
    text_align_parser.add_argument("--ids_file", "-I", default='',
                                   help = "use only those ids, otherwise process everything")
    text_align_parser.set_defaults(func = create_text_align_task)

    sent_retr_parser = subparsers.add_parser('sent_retr')
    sent_retr_parser.add_argument("--subm_dir", "-i", required = True,
                                   help = "directory with submissions")
    sent_retr_parser.add_argument("--out_prefix", "-o", required=True)
    sent_retr_parser.add_argument("--ids_file", "-I", default='',
                                   help = "use only those ids, otherwise process everything")
    sent_retr_parser.set_defaults(func = create_sent_retrieval_meta)

    src_retr_parser = subparsers.add_parser('src_retr',
                                            help='help of src_retr')
    src_retr_parser.add_argument("--subm_dir", "-i", required = True,
                                 help = "directory with submissions")
    src_retr_parser.add_argument("--src_retr_out_dir", "-o", default="src_retrieval")
    src_retr_parser.add_argument("--mapping", "-m", default = "src_mapping.csv",
                                 help = "mapping file path")
    src_retr_parser.add_argument("--min_sent_cnt", "-s", default=4, type=int)
    src_retr_parser.add_argument("--use_filename_as_id", "-u", action='store_true')
    src_retr_parser.add_argument("--ids_file", "-I", default='',
                                 help = "use only those ids, otherwise process everything")
    src_retr_parser.set_defaults(func = create_src_retr_meta)

    dos_sim_parser = subparsers.add_parser('doc_sim',
                                           help='help of src_retr')
    dos_sim_parser.add_argument("--subm_dir", "-i", required = True,
                                help = "directory with submissions")
    dos_sim_parser.add_argument("--src_retr_out_dir", "-o", default="doc_sim")
    dos_sim_parser.add_argument("--mapping", "-m", default = "src_mapping.csv",
                                help = "mapping file path")
    dos_sim_parser.add_argument("--min_sent_cnt", "-s", default=2, type=int)
    dos_sim_parser.add_argument("--use_filename_as_id", "-u", action='store_true')
    dos_sim_parser.add_argument("--ids_file", "-I", default='',
                                help = "use only those ids, otherwise process everything")
    dos_sim_parser.set_defaults(func = create_doc_sim_meta)

    pan_parser = subparsers.add_parser('pan',
                                       help='help of pan')
    pan_parser.add_argument("--subm_dir", "-i", required = True,
                            help = "directory with submissions")
    pan_parser.add_argument("--mapping", "-m", default = "src_mapping.csv",
                            help = "mapping file path")
    pan_parser.add_argument("--src_retr_out_dir", "-s", default="src_retrieval")
    pan_parser.add_argument("--min_sent_cnt", "-c", default=4, type=int)
    pan_parser.add_argument("--use_filename_as_id", "-u", action='store_true')
    pan_parser.add_argument("--text_align_out_dir", "-t",
                            default="01-manual-plagiarism")
    pan_parser.add_argument("--ids_file", "-I", default='',
                            help = "use only those ids, otherwise process everything")
    pan_parser.set_defaults(func = create_pan_meta)
    args = parser.parse_args()

    FORMAT="%(asctime)s %(levelname)s: %(name)s: %(message)s"
    logging.basicConfig(level = logging.DEBUG if args.verbose else logging.INFO,
                        format = FORMAT)
    try:

        args.func(args)
    except Exception as e:
        logging.exception("failed to gen: %s ", e)

if __name__ == '__main__' :
    main()

class Opts(object):
    def __init__(self):
        self.version = None

def test():

    pipes = [DumbDumper()]
    gener = Generator(Opts(), pipes)
    gener.process_archive("data/test_data/test.zip", "4")

def test_v2():
    pipes = [DumbDumper()]
    gener = Generator(Opts(), pipes)
    gener.process_archive("data/test_data_v2/test.zip", "2000")


import zipfile
from io import BytesIO
from unittest.mock import patch

import pytest
from mock_module.config import ServiceWithFilesConfig
from mock_module.resource import (
    CustomDisabledUploadFileResourceConfig,
    CustomFileResourceConfig,
    CustomRecordResourceConfig,
)
from zipstream import ZipStream

from invenio_records_resources.resources import FileResource, RecordResource
from invenio_records_resources.services import RecordService

@pytest.fixture(scope="module")
def service():
    return RecordService(ServiceWithFilesConfig)

@pytest.fixture(scope="module")
def record_resource(service):
    """Record Resource."""
    return RecordResource(CustomRecordResourceConfig, service)

@pytest.fixture(scope="module")
def file_resource(file_service):
    """File Resources."""
    return FileResource(CustomFileResourceConfig, file_service)

@pytest.fixture(scope="module")
def disabled_file_upload_resource(file_service):
    """Disabled Upload File Resource."""
    return FileResource(CustomDisabledUploadFileResourceConfig, file_service)

@pytest.fixture(scope="module")
def base_app(
    base_app, file_resource, disabled_file_upload_resource, service, file_service
):
    """Application factory fixture."""
    base_app.register_blueprint(file_resource.as_blueprint())
    base_app.register_blueprint(disabled_file_upload_resource.as_blueprint())
    registry = base_app.extensions["invenio-records-resources"].registry
    registry.register(service, service_id="mock-records")
    registry.register(file_service, service_id="mock-files")
    yield base_app

@pytest.fixture()
def input_data(input_data):
    input_data["files"] = {"enabled": True}
    return input_data

def test_files_api_flow(client, search_clear, headers, input_data, location):
    """Test record creation."""
    # Initialize a draft
    res = client.post("/mocks", headers=headers, json=input_data)
    assert res.status_code == 201
    id_ = res.json["id"]
    assert res.json["links"]["files"].endswith(f"/api/mocks/{id_}/files")

    # Initialize files upload
    res = client.post(
        f"/mocks/{id_}/files",
        headers=headers,
        json=[
            {"key": "test.pdf", "title": "Test file"},
        ],
    )
    assert res.status_code == 201
    res_file = res.json["entries"][0]
    assert res_file["key"] == "test.pdf"
    assert res_file["status"] == "pending"
    assert res_file["metadata"] == {"title": "Test file"}
    assert res_file["links"]["self"].endswith(f"/api/mocks/{id_}/files/test.pdf")
    assert res_file["links"]["content"].endswith(
        f"/api/mocks/{id_}/files/test.pdf/content"
    )
    assert res_file["links"]["commit"].endswith(
        f"/api/mocks/{id_}/files/test.pdf/commit"
    )

    # Get the file metadata
    res = client.get(f"/mocks/{id_}/files/test.pdf", headers=headers)
    assert res.status_code == 200
    assert res.json["key"] == "test.pdf"
    assert res.json["status"] == "pending"
    assert res.json["metadata"] == {"title": "Test file"}

    # Upload a file
    res = client.put(
        f"/mocks/{id_}/files/test.pdf/content",
        headers={
            "content-type": "application/octet-stream",
            "accept": "application/json",
        },
        data=BytesIO(b"testfile"),
    )
    assert res.status_code == 200
    assert res.json["status"] == "pending"

    # Commit the uploaded file
    res = client.post(f"/mocks/{id_}/files/test.pdf/commit", headers=headers)
    assert res.status_code == 200
    assert res.json["status"] == "completed"

    # Get the file metadata
    res = client.get(f"/mocks/{id_}/files/test.pdf", headers=headers)
    assert res.status_code == 200
    assert res.json["key"] == "test.pdf"
    assert res.json["status"] == "completed"
    assert res.json["metadata"] == {"title": "Test file"}
    file_size = str(res.json["size"])
    assert isinstance(res.json["size"], int), "File size not integer"

    # Read a file's content
    res = client.get(f"/mocks/{id_}/files/test.pdf/content", headers=headers)
    assert res.status_code == 200
    assert res.data == b"testfile"

    # Update file metadata
    res = client.put(
        f"/mocks/{id_}/files/test.pdf", headers=headers, json={"title": "New title"}
    )
    assert res.status_code == 200
    assert res.json["key"] == "test.pdf"
    assert res.json["status"] == "completed"
    assert res.json["metadata"] == {"title": "New title"}

    # Get all files
    res = client.get(f"/mocks/{id_}/files", headers=headers)
    assert res.status_code == 200
    assert len(res.json["entries"]) == 1
    assert res.json["entries"][0]["key"] == "test.pdf"
    assert res.json["entries"][0]["status"] == "completed"
    assert res.json["entries"][0]["metadata"] == {"title": "New title"}

    # Delete a file
    res = client.delete(f"/mocks/{id_}/files/test.pdf", headers=headers)
    assert res.status_code == 204

    # Get all files
    res = client.get(f"/mocks/{id_}/files", headers=headers)
    assert res.status_code == 200
    assert len(res.json["entries"]) == 0

def test_empty_file(client, search_clear, headers, input_data, location):
    """Test if an empty file works properly."""
    # Initialize a draft
    res = client.post("/mocks", headers=headers, json=input_data)
    assert res.status_code == 201
    id_ = res.json["id"]
    assert res.json["links"]["files"].endswith(f"/api/mocks/{id_}/files")

    # Initialize files upload
    res = client.post(
        f"/mocks/{id_}/files",
        headers=headers,
        json=[
            {"key": "empty", "title": "Zero-length test file"},
        ],
    )
    assert res.status_code == 201

    # Upload the empty file
    res = client.put(
        f"/mocks/{id_}/files/empty/content",
        headers={
            "content-type": "application/octet-stream",
            "accept": "application/json",
        },
        data=BytesIO(b""),
    )
    assert res.status_code == 200

    # Commit the uploaded file
    res = client.post(f"/mocks/{id_}/files/empty/commit", headers=headers)
    assert res.status_code == 200

    # Get the file metadata, check if everything (especially size) is present
    res = client.get(f"/mocks/{id_}/files/empty", headers=headers)
    assert res.status_code == 200
    assert res.json["key"] == "empty"
    assert res.json["checksum"] == "md5:d41d8cd98f00b204e9800998ecf8427e"
    assert res.json["size"] == 0

    # Read a file's content
    res = client.get(f"/mocks/{id_}/files/empty/content", headers=headers)
    assert res.status_code == 200
    assert res.data == b""

def test_default_preview_file(app, client, search_clear, headers, input_data, location):
    # Initialize a draft
    res = client.post("/mocks", headers=headers, json=input_data)
    assert res.status_code == 201
    id_ = res.json["id"]
    assert res.json["links"]["files"].endswith(f"/api/mocks/{id_}/files")

    # Initialize 3 file uploads
    res = client.post(
        f"/mocks/{id_}/files",
        headers=headers,
        json=[
            {"key": "f1.pdf"},
            {"key": "f2.pdf"},
            {"key": "f3.pdf"},
        ],
    )
    assert res.status_code == 201
    file_entries = res.json["entries"]
    assert len(file_entries) == 3
    assert {(f["key"], f["status"]) for f in file_entries} == {
        ("f1.pdf", "pending"),
        ("f2.pdf", "pending"),
        ("f3.pdf", "pending"),
    }
    assert res.json["default_preview"] is None

    # Upload and commit the 3 files
    for f in file_entries:
        res = client.put(
            f"/mocks/{id_}/files/{f['key']}/content",
            headers={
                "content-type": "application/octet-stream",
                "accept": "application/json",
            },
            data=BytesIO(b"testfile"),
        )
        assert res.status_code == 200
        assert res.json["status"] == "pending"

        res = client.post(f"/mocks/{id_}/files/{f['key']}/commit", headers=headers)
        assert res.status_code == 200
        assert res.json["status"] == "completed"

    # Set the default preview file
    input_data["files"]["default_preview"] = "f1.pdf"
    res = client.put(f"/mocks/{id_}", headers=headers, json=input_data)
    assert res.status_code == 200
    assert res.json["files"]["default_preview"] == "f1.pdf"

    # Change the default preview file
    input_data["files"]["default_preview"] = "f2.pdf"
    res = client.put(f"/mocks/{id_}", headers=headers, json=input_data)
    assert res.status_code == 200
    assert res.json["files"]["default_preview"] == "f2.pdf"

    # Unset the default preview file
    input_data["files"]["default_preview"] = None
    res = client.put(f"/mocks/{id_}", headers=headers, json=input_data)
    assert res.status_code == 200
    assert res.json["files"].get("default_preview") is None

    # Empty string the default preview file
    input_data["files"]["default_preview"] = ""
    res = client.put(f"/mocks/{id_}", headers=headers, json=input_data)
    assert res.status_code == 200
    assert res.json["files"].get("default_preview") is None

    # Set the default preview file
    input_data["files"]["default_preview"] = "f3.pdf"
    res = client.put(f"/mocks/{id_}", headers=headers, json=input_data)
    assert res.status_code == 200
    assert res.json["files"]["default_preview"] == "f3.pdf"

    # Delete the default preview file
    res = client.delete(f"/mocks/{id_}/files/f3.pdf", headers=headers)
    assert res.status_code == 204

    # Get all files and check default preview
    res = client.get(f"/mocks/{id_}/files", headers=headers)
    assert res.status_code == 200
    assert len(res.json["entries"]) == 2
    assert res.json["default_preview"] is None

def test_file_api_errors(client, search_clear, headers, input_data, location):
    """Test REST API errors for file management."""
    h = headers

    # Initialize a draft
    res = client.post("/mocks", headers=headers, json=input_data)
    assert res.status_code == 201
    id_ = res.json["id"]
    assert res.json["links"]["files"].endswith(f"/api/mocks/{id_}/files")

    # Initialize files upload
    # Pass an object instead of an array
    res = client.post(f"/mocks/{id_}/files", headers=headers, json={"key": "test.pdf"})
    assert res.status_code == 400

    res = client.post(
        f"/mocks/{id_}/files",
        headers=headers,
        json=[
            {"key": "test.pdf", "title": "Test file"},
        ],
    )
    assert res.status_code == 201

    # Upload a file
    res = client.put(
        f"/mocks/{id_}/files/test.pdf/content",
        headers={
            "content-type": "application/octet-stream",
            "accept": "application/json",
        },
        data=BytesIO(b"testfile"),
    )
    assert res.status_code == 200
    assert res.json["status"] == "pending"

    # Commit the uploaded file
    res = client.post(f"/mocks/{id_}/files/test.pdf/commit", headers=headers)
    assert res.status_code == 200
    assert res.json["status"] == "completed"

    # Initialize same file upload again
    res = client.post(
        f"/mocks/{id_}/files",
        headers=headers,
        json=[
            {"key": "test.pdf", "title": "Test file"},
        ],
    )
    assert res.status_code == 400

def test_disabled_upload_file_resource(
    client, search_clear, headers, input_data, location
):
    """Test file resources with disabled file upload"""

    # Initialize a draft
    res = client.post("/mocks", headers=headers, json=input_data)
    assert res.status_code == 201
    id_ = res.json["id"]

    # File manipulation api should not be exposed if allow_upload is disabled
    res = client.post(
        f"/mocks_disabled_files_upload/{id_}/files",
        headers=headers,
        json=[{"key": "test.pdf", "title": "Test file"}],
    )
    assert res.status_code == 405

    res = client.put(
        f"/mocks_disabled_files_upload/{id_}/files",
        headers=headers,
        json=[{"default_preview": "test.pdf"}],
    )
    assert res.status_code == 405

    res = client.delete(
        f"/mocks_disabled_files_upload/{id_}/files/test.pdf", headers=headers
    )
    assert res.status_code == 405

def test_disable_files_when_files_already_present_should_error(
    app, client, search_clear, headers, input_data, location
):
    # Initialize a record
    response = client.post("/mocks", headers=headers, json=input_data)
    id_ = response.json["id"]
    # Add file
    file_id = "f1.pdf"
    client.post(f"/mocks/{id_}/files", headers=headers, json=[{"key": file_id}])
    client.put(
        f"/mocks/{id_}/files/{file_id}/content",
        headers={
            "content-type": "application/octet-stream",
            "accept": "application/json",
        },
        data=BytesIO(b"testfile"),
    )
    client.post(f"/mocks/{id_}/files/{file_id}/commit", headers=headers)
    # Disable files
    input_data["files"] = {"enabled": False}

    response = client.put(f"/mocks/{id_}", headers=headers, json=input_data)

    assert response.status_code == 400
    assert response.json["errors"] == [
        {
            "field": "files.enabled",
            "messages": [
                "You must first delete all files to set the record to be "
                "metadata-only."
            ],
        }
    ]

def test_download_archive(
    app,
    client,
    search_clear,
    headers,
    input_data,
    location,
):
    # Initialize a draft
    res = client.post("/mocks", headers=headers, json=input_data)
    assert res.status_code == 201
    id_ = res.json["id"]
    assert res.json["links"]["files"].endswith(f"/api/mocks/{id_}/files")

    # Initialize 3 file uploads
    res = client.post(
        f"/mocks/{id_}/files",
        headers=headers,
        json=[
            {"key": "f1.pdf"},
            {"key": "f2.pdf"},
            {"key": "f3.pdf"},
        ],
    )
    assert res.status_code == 201
    file_entries = res.json["entries"]
    assert len(file_entries) == 3
    assert {(f["key"], f["status"]) for f in file_entries} == {
        ("f1.pdf", "pending"),
        ("f2.pdf", "pending"),
        ("f3.pdf", "pending"),
    }

    # Upload and commit the 3 files
    for f in file_entries:
        res = client.put(
            f"/mocks/{id_}/files/{f['key']}/content",
            headers={
                "content-type": "application/octet-stream",
                "accept": "application/json",
            },
            data=BytesIO(b"testfile"),
        )
        assert res.status_code == 200
        assert res.json["status"] == "pending"

        res = client.post(f"/mocks/{id_}/files/{f['key']}/commit", headers=headers)
        assert res.status_code == 200
        assert res.json["status"] == "completed"

    captured_fps = []

    class MockZipStream(ZipStream):
        def add(self, fp, *args, **kwargs):
            # Keep track of all passed file pointers
            captured_fps.append(fp)
            return super().add(fp, *args, **kwargs)

    # Get all files as zipped archive
    with patch(
        "invenio_records_resources.resources.files.resource.ZipStream",
        new=MockZipStream,
    ):
        res = client.get(
            f"/mocks/{id_}/files-archive",
        )
        assert res.status_code == 200
        res_bytes = BytesIO(res.data)
        with zipfile.ZipFile(res_bytes, "r") as zf:
            files = zf.namelist()
            files.sort()
            assert files == ["f1.pdf", "f2.pdf", "f3.pdf"]
    assert all(f.closed for f in captured_fps)


# imports
import pandas as pd
from dict2xml import dict2xml
import yaml
import json

data = pd.read_csv("input.csv")

# Calculate AVG
def Average(lst):
    return round(sum(lst) / len(lst),2)

def dict_to_xml(agg_dict):
    """Convert to xml."""
    xml = dict2xml(agg_dict)
    return xml

def write_to_xml(file_name,data):
    """Write to xml."""
    with open(file_name,'w') as f:
        f.write(data)

def write_to_yaml(file_name,data):
    """Write to yaml."""
    with open(file_name,'w') as f:
        yaml.dump(data, f)

def write_to_json(file_name,data):
    """Write to json."""
    with open(file_name, "w") as f:
        json.dump(data, f)

# calculate the stats
def stats():
    """Calculate the summary."""
    agg_values = []
    total_sum=total_count=0
    agg_min=float('inf')
    agg_max=-float('inf')
    agg_average=0
    summary = {
        "hosts":[],
        "agg_min":0,
        "agg_max":0,
        "agg_avg":0
    }
    print("HOST_NAME, MAX, MIN, AVG")
    for column in data:
        column_obj = data[column]
        if "#" in column:
            host_name,host_max,host_min,host_avg=column.split("#")[1],column_obj.values.max(), column_obj.values.min() , Average(column_obj.values)
            print(column.split("#")[1], column_obj.values.max(), column_obj.values.min() , Average(column_obj.values))
            # Cleanup the missing values
            values=[x for x in column_obj.values if str(x) != 'nan']
            total_sum+=sum(values)
            total_count+=len(values)
            agg_max=max(agg_max,host_max)
            agg_min=min(agg_min,host_min)
            # convert nan to 'nan' (to generate yaml successfully)
            host_map={"max": str(host_max) if str(host_max) == 'nan' else int(host_max),"min": str(host_min) if str(host_min) == 'nan' else int(host_min),"average":str(host_avg) if str(host_avg) == 'nan' else int(host_avg),"name":host_name}
            summary["hosts"].append(host_map)

    summary["agg_max"]=float(agg_max)
    summary["agg_min"]=float(agg_min)
    summary["agg_avg"]=float(round(total_sum/total_count,2))
    return summary

# main
def main():
    print("This program calculates the memory usage")
    # find the summary
    output = stats()
    #convert to xml
    xml=dict_to_xml(output)
    #write to xml
    write_to_xml('output.xml',xml)
    #write to json
    write_to_json('output.json',output)
    #write to yaml
    write_to_yaml('output.yaml',output)

main()


#import libraries
import numpy as np
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import keras
import tensorflow as tf
#%%
#import dataset
fashion_mnist = keras.datasets.fashion_mnist
(X_train, Y_train), (X_test, Y_test) = fashion_mnist.load_data()
#%%
#setting labels
class_names =np.array(["T-shirt/top", "Trouser", "Pullover", "Dress", "Coat", "Sandal", "Shirt", "Sneaker", "Bag", "Ankle Boot"])
#%%
#view data
plt.imshow(X_train[10])  #imshow - image show
print(class_names[Y_train[10]])
#%%
#normalisation (by 255 since grayscale is 0 to 255)
X_train = X_train /255.0
X_test = X_test /255.0
#%%
#train set to train and validation set
#validation set model optimization
X_valid, X_train = X_train[:5000], X_train[5000:]
Y_valid, Y_train = Y_train[:5000], Y_train[5000:]
#%%
#set randomseed as 42
np.random.seed(42)
tf.random.set_seed(42)
#%%
#model description
#input layer - 28*28 pixel image
#two hidden layers with RELU activation -300 and 100 neurons respectively in layers
#output layer - 10 classes with softmax activation
#%%
#building model
model = keras.models.Sequential()
model.add(keras.layers.Flatten(input_shape=(28, 28)))   #flatten the images in input layer
model.add(keras.layers.Dense(300, activation = 'relu'))
model.add(keras.layers.Dense(100, activation = 'relu'))
model.add(keras.layers.Dense(10, activation='softmax'))
print(model.summary())
#%%
#visualise model
from keras.utils import plot_model
plot_model(model, to_file='model_plot1.png',show_shapes=True)
#%%
#get weights
w, b = model.layers[1].get_weights()
#%%
#learning
#since Y is labels using sparse_categorical_crossentropy (if Y is probability of class use categorical_crossentropy)
#if Y is T/F use binary_crossentropy
model.compile(loss="sparse_categorical_crossentropy", optimizer="sgd", metrics=['accuracy'])
#%%
#early stopping
checkpts = keras.callbacks.ModelCheckpoint("Early_Stop-Save.h5", save_best_only=True)
#save_best_only - saves only model with best val accuracy
early_stop = keras.callbacks.EarlyStopping(patience=10, restore_best_weights=True)
#patience - stop model training if val accuracy is not improving for 10 epochs continuously
#%%
#training
model_history = model.fit(X_train, Y_train, epochs=500, validation_data=(X_valid, Y_valid), callbacks=[checkpts, early_stop])
#%%
#visualising the training
pd.DataFrame(model_history.history).plot()
plt.grid(True)
plt.gca().set_ylim(0,1)
plt.show()
#%%
#model evaluation
eval_hist = model.evaluate(X_test, Y_test)
#first value - loss
#second value - accuracy
#%%
#prediction
Y_pred = model.predict(X_test)  #probability of the image to belong in the classes
#Y_pred = Y_pred.round(2)        #round the probability to two places
Y_pred_class = model.predict_classes(X_test)
Y_pred_class = class_names[Y_pred_class]
#%%
#verifying manually
plt.imshow(X_test[0])
print(Y_pred_class[0])
#%%

Supervised Learning/Fashion-MNIST-Classification-ANN/2) Early Stopping/MNIST image Classification.py