# coding: UTF-8
from core import app
from flask import Markup, redirect, render_template, request, url_for
from models import Post, PostTags, Tag
from utils import slugify

import markdown

@app.route('/')
def index():
    """
    Renders the main page and displays
    all the posts
    """
    tags = Tag.all().order('-name')
    posts = Post.all().order('-created')

    return render_template('index.html', posts=posts, list_of_tags=tags)

@app.route('/<slug>')
def view_post(slug):
    """
    Renders a view to a specific post
    :params:slug - the post slug is used to get the post
    """
    post = Post.by_slug(slug)
    return render_template('viewpost.html', post=post)

@app.route('/admin/_newpost/', methods=['GET', 'POST'])
def new_post():
    """
    Renders a view to create new posts
    """
    # get all tags in the model
    stored_tags = Tag.all()
    if request.method == 'POST':
        title = request.form['title']

        # stores tags as a list
        tags = request.form.getlist('tags')

        # stores markdown without HTML formatting
        markdown_content = request.form['content']

        # stores markdown formatted as HTML
        html_content = Markup(markdown.markdown(markdown_content))
        slug = slugify(title)

        params = dict(title=title, markdown_text=markdown_content,
                      tags=tags)

        # if the post has a title, slug and content
        if title and markdown_content:
            pt = Post(title=title,
                      html_text=html_content,
                      markdown_text=markdown_content,
                      slug=slug)

            # save the post
            pt.put()

            # once save use the post reference key (pt) to save the post tags
            PostTags(post=pt, name=tags).put()

            return redirect(url_for('index'))
        else:
            params['error_post'] = ("Please make sure you have a title,"
                                    " and content")
            return render_template('newpost.html',
                                   stored_tags=stored_tags,
                                   **params)
    else:
        # if its a request method is a GET
        return render_template('newpost.html', stored_tags=stored_tags)

@app.route('/admin/_edit/<slug>/', methods=['GET', 'POST'])
def edit_post(slug):
    """
    Renders a view to edit specific posts
    :params: slug- the post slug is used to get the post
    """
    if request.method == 'GET':
        post = Post.by_slug(slug)

        # get all tags in the model, to allow the user to
        # add more tags to the post
        tags = Tag.all().order('-name')
        return render_template('editpost.html', post=post, list_of_tags=tags)
    else:
        update_post = Post.by_slug(slug)

        update_or_add_tags = PostTags.all().filter(
            'post =', update_post.key()).get()

        new_title = request.form['title']
        new_markdown_text = request.form['content']
        # list of stored tags
        stored_tags = request.form.getlist('stored-tags')
        # list of new tags
        new_tags = request.form.getlist('new-tags')
        # concatenate the two lists
        list_tags = stored_tags + new_tags
        # update the current post title, slug, markdown text and html text
        update_post.title = new_title
        update_post.markdown_text = new_markdown_text
        update_post.html_text = Markup(markdown.markdown(new_markdown_text))
        update_post.slug = slugify(new_title)
        # save the changes
        update_post.put()

        if update_or_add_tags:
            # update/add the tags in current post
            update_or_add_tags.name = list(set(list_tags))
        else:
            update_or_add_tags = PostTags(post=update_post,
                                          name=list(set(list_tags)))

        # save the changes
        update_or_add_tags.put()

        return redirect(url_for('view_post', slug=slug))

@app.route('/admin/_tags/')
def tags():
    """
    Renders a view to add new tags, and edit existing tags
    """
    tags = Tag.all().order('-name')
    return render_template('tags.html', tags=tags)

@app.route('/admin/_newtag/', methods=['GET', 'POST'])
def new_tag():
    """
    Renders a view to add new tags
    """
    if request.method == 'POST':
        tag_name = request.form['tag']
        # description = request.form['description']
        has_errors = False
        params = dict(tag=tag_name)

        # check if the tag exists first
        tag_exists = Tag.all().filter('name =', tag_name).get()
        if not tag_name:
            # if the user submits an empty string
            params['error_tag'] = ('Please make sure you have a Tag Name')
            has_errors = True
        if tag_exists:
            # if the user tries to submit an existing Tag
            params['existing_tag'] = (
                'The tag: {} already exists!'.format(tag_exists.name)
            )
            has_errors = True

        if has_errors:
            return render_template('newtag.html', **params)
        else:
            tag = Tag(name=tag_name, slug=tag_name, description='')
            tag.put()
            return redirect(url_for('tags'))
    else:
        return render_template('newtag.html')

@app.route('/admin/_delete_tag/<slug>/', methods=['GET', 'POST'])
def delete_tag(slug):
    if request.method == 'GET':
        post_slug = Tag.all().filter('slug =', slug).get()
        post_slug.delete()
        return redirect(url_for('tags'))


# -*- coding: utf-8 -*-
"""
Created on Fri Jan  3 23:25:55 2020

@author: AKASH DIXIT
"""

import http.client, urllib.request, urllib.parse, urllib.error, base64
subscription_key = 'xxxxxxxxxxxxxxxxxxxxxxxxx'
headers = {
    # Request headers
    'Content-Type': 'application/json',
    'Ocp-Apim-Subscription-Key':subscription_key
}

body = {
    "faceId": "37a6b4dc-bb24-4d68-90f4-0dd3067498f9",
    "personId": "7cc20792-2367-4ad2-b046-2073478bc5d7",
    "personGroupId": "friends"
    }

params = urllib.parse.urlencode({
})

try:
    conn = http.client.HTTPSConnection('centralindia.api.cognitive.microsoft.com')
    conn.request("POST", "/face/v1.0/verify?%s" % params, str(body), headers)
    response = conn.getresponse()
    data = response.read()
    print(data)
    conn.close()
except Exception as e:
    print("[Errno {0}] {1}".format(e.errno, e.strerror))


import yaml
from yaml import load, load_all

stream = open('sample.yaml', 'r')
document = load_all(stream, Loader=yaml.FullLoader)

print(type(document))

for doc in document:
	print(type(doc))

	print(doc['people'][1])


import numpy as np
import cv2
import extcolors
from matplotlib import pyplot as plt

path1 = "shape1.jpg"
# path2 = "shape2.jpg"
path2 = "shape32.jpg"
# path2 = "shape1 - Copy.jpg"
# path1 = "shape31.jpg"
# path1 = "apple_logo.jpg"
path = "img.jpg"

def bgExtract(path):
    img = cv2.imread(path, 1)
    colors, pixel_count = extcolors.extract(path)
    # print(path)
    normPercentages = []
    pieColor = []
    colorLabels = []

    for i in range(len(colors)):
        # keep here the rgb code of the color
        colorRGB = str(colors[i][0])
        colorLabels.append(colorRGB)
#       calculate the percentage of the rgb color in the picture
        percentage = colors[i][1]/pixel_count * 100
        # format the percentage and put it in the matrix of the original percentages
        colorPercentage = '{0:.3f}'.format(percentage)
        normPercentages.append(colorPercentage)

        # We normalize some small values so that they appear correclty in the pie chart
        if percentage <= 0.01:
            percentage = percentage*300
            # print("IN IF ---- with % ",percentage)
        elif percentage <= 0.1:
            percentage = percentage*30
            # print("elif ---- with % ",percentage)
        elif percentage <= 0.5:
            percentage = percentage *10

        # We enter those corrected percentages in the list of the pie chart sizes(how big is each chunk)
        pieColor.append(percentage)
        colorPercentage = '{0:.3f}'.format(percentage)
        # print(colorRGB + ": " + colorPercentage)

    # create the pie chart
    plt.figure()
    patches,texts = plt.pie(pieColor,labels=normPercentages)

    plt.legend(patches, colorLabels, loc="best")

    cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
    canny = cv2.Canny(img, 100, 200)
    cv2.imshow(path, canny)

    contours,hierarchy = cv2.findContours(canny,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)
    if len(contours) > 0:
        cnt = contours[0]
        area = cv2.contourArea(cnt)
        print(area)
        print("\n")

    return area

area1 = bgExtract(path1)
area2 = bgExtract(path2)
if area1 > area2:
    ratio = area1/area2
    pathStr = path1 + "/" + path2
    print("The "+ pathStr +" ratio ---> {0:.2f}".format(ratio))

else:
    ratio = area2 / area1
    pathStr = path2 + "/" + path1
    print("The " + pathStr + " ratio ---> {0:.2f}".format(ratio))

# bgExtract(path)
plt.show("The colors")
cv2.waitKey()


import sys, os, csv
import django
os.environ['DJANGO_SETTINGS_MODULE'] = 'se_project.settings'
django.setup()
#project_dir = "/home/HOMEGROUP/adelg000/school/CEN4010/BookStore/BookStore/"
#sys.path.append(project_dir)

from user.models import *

count = 0
data = csv.reader(open("sample_data/CART_CONTENT.csv"),delimiter=",")
for row in data:

	if count == 1:
		print("Reading CART_CONTENT model ...")

		cc = CART_CONTENT()

		cc.Cart_contentID=int(row[0])
		print("Cart_contentID = " + str(cc.Cart_contentID))

		# Reading ISBN ...
		bList = BOOK.objects.filter(ISBN=row[1])
		if bList:
			b = bList[0]

			if b:
				cc.ISBN = b;
		else:
			b = BOOK()
			b.ISBN = row[1]
			b.save()
			cc.ISBN = b
		print("ISBN = " + cc.ISBN.ISBN)

		# Reading Cart_ID ...
		cList = CART.objects.filter(Cart_ID=row[2])
		if cList:
			c = cList[0]

			if c:
				cc.Cart_ID = c;
		else:
			c = CART()
			c.Cart_ID = row[2]
			c.save()

		print("Cart_ID = " + cc.Cart_ID.Cart_ID)

		cc.Quantity = int(row[3])
		print("Quantity = " + str(cc.Quantity))

		cc.save()
		print("CART_CONTENT read SUCCESSFUL")

	count = 1


from dependency_injector.wiring import inject, Provide
from fastapi import APIRouter, Depends, Response

from schemas import application_schemas
from services.application_service import ApplicationService
from services.change_history_service import ChangeHistoryService
from containers import Container

router = APIRouter(tags=['applications'])

@router.post("/applications", response_model=application_schemas.ApplicationSchema, status_code=201)
@inject
async def create(
    app: application_schemas.ApplicationCreateSchema,
    app_service: ApplicationService = Depends(Provide[Container.app_service])
) -> Response:
    """Creates an application

    """

    app = await app_service.create(app)

    return app

@router.put("/applications/{app_id}", response_model=application_schemas.ApplicationSchema)
@inject
async def update(
    app_id: int,
    app_data: application_schemas.ApplicationCreateSchema,
    app_service: ApplicationService = Depends(Provide[Container.app_service]),
    change_hostory_service: ChangeHistoryService = Depends(Provide[Container.change_history_service])
) -> Response:
    """Updates an application by id

    """

    await change_hostory_service.make_history(app_id, 'applications', app_data)

    return await app_service.update(app_id, app_data)

@router.delete("/applications/{app_id}", status_code=200)
@inject
async def delete(
    app_id: int,
    app_service: ApplicationService = Depends(Provide[Container.app_service])
) -> Response:
    """Deletes an application by id

    """

    await app_service.delete(app_id=app_id)

    return {'deleted': app_id}

@router.get("/applications", response_model=application_schemas.ApplicationsListSchema)
@inject
async def get_list(
    page: int = 1,
    per_page: int = 10,
    app_service: ApplicationService = Depends(Provide[Container.app_service])
) -> Response:
    """Gets all existing applications

    """

    total_count = await app_service.get_count()
    applications = await app_service.get_list(page, per_page)

    return {"total_count": total_count, "data": applications}


from django.urls import path

from . import views

urlpatterns = [
    path("<int:id>", views.index, name="index"),
    path("", views.home, name="home"),
    path("home/", views.home, name="home"),
    path("create/", views.create, name="index"),
    path("view/", views.view, name="view"),
    path("assets/", views.assets, name="assets"),
    path("softwares/", views.softwares, name="softwares")
]


import redis
import time
import random

def add_height_score(db0, db1):
    for player_id in db1.keys():
        height = int(float(db0.lindex(player_id, 5).decode("utf-8")) / 2)
        db1.incr(player_id, height)

def main():
    db0 = redis.Redis(host="localhost", port=6379, db=0)
    db1 = redis.Redis(host="localhost", port=6379, db=1)

    while True:
        try:
            for player_id in db1.keys():
                height = int(float(db0.lindex(player_id, 5).decode("utf-8")))
                db1.incr(player_id, height)

            time.sleep(10)

        # 処理中にプレイヤーが死んだ場合
        except:
            time.sleep(10)

if __name__ == "__main__":
    main()


#!/usr/bin/env python3
"""
Create a standalone telepresence
"""

import sys
from io import BytesIO
from pathlib import Path
from subprocess import check_output
from tempfile import TemporaryDirectory
from zipapp import create_archive
from zipfile import ZIP_DEFLATED, ZipFile

def make_compressed_zipapp(source: Path, output: Path, main_function: str):
    """
    Make an executable compressed zipfile from Python code
    """
    # Build uncompresed zip file
    uncompressed = BytesIO()
    create_archive(source, uncompressed, main=main_function)
    uncompressed.seek(0, 0)

    # Build compressed zip file
    compressed = BytesIO()
    in_zip = ZipFile(uncompressed)
    with ZipFile(compressed, mode="w") as out_zip:
        for info in in_zip.infolist():
            out_zip.writestr(info, in_zip.read(info), ZIP_DEFLATED)
    compressed.seek(0, 0)

    # Build final zip file
    create_archive(compressed, str(output), interpreter="/usr/bin/env python3")
    output.chmod(0o755)  # Seems to default to 744 on CircleCI

def build_telepresence(project: Path, output: Path):
    """
    Build a telepresence executable zipfile
    """
    with TemporaryDirectory() as temp:
        # Run setup.py build to extract source code. Versioneer will replace
        # _version.py with hard-coded version info.
        check_output(["python3", "-Wignore", "setup.py", "build", "-b", temp],
                     cwd=str(project))
        # Build zip app
        make_compressed_zipapp(
            Path(temp) / "lib", output, "telepresence.main:run_telepresence"
        )

    print("Built {}".format(output))

def main():
    """
    Set things up then call the code that builds the executable.
    """
    project = Path(__file__).absolute().resolve().parent.parent

    if len(sys.argv) > 1:
        output = Path(sys.argv[1])
    else:
        version_bytes = check_output(
            ["python3", "-Wignore", "setup.py", "--version"],
            cwd=str(project),
        )
        version = str(version_bytes, "utf-8").strip()

        output = project / "dist" / "telepresence-{}".format(version)

    output.parent.mkdir(parents=True, exist_ok=True)
    if output.exists():
        output.unlink()

    build_telepresence(project, output)

if __name__ == "__main__":
    main()


# Copyright 2014 Openstack Foundation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import json

from neutron.openstack.common import log as logging
from oslo.config import cfg
import XenAPI

LOG = logging.getLogger(__name__)
CONF = cfg.CONF

agent_opts = [
    cfg.StrOpt("xapi_connection_url"),
    cfg.StrOpt("xapi_connection_username", default="root"),
    cfg.StrOpt("xapi_connection_password")
]

CONF.register_opts(agent_opts, "AGENT")

class VIF(object):
    SEPARATOR = "."

    def __init__(self, device_id, mac_address):
        """Constructs VIF

        `device_id` and `mac_address` should be strings if they will later be
        compared to decoded VIF instances (via from_string).
        """

        self.device_id = device_id
        self.mac_address = mac_address

    def __str__(self):
        return "%s%s%s" % (self.device_id, self.SEPARATOR, self.mac_address)

    @classmethod
    def from_string(cls, s):
        device_id, mac_address = s.split(cls.SEPARATOR)
        return cls(device_id, mac_address)

    def __repr__(self):
        return "VIF(%r, %r)" % (self.device_id, self.mac_address)

    def __eq__(self, other):
        return (self.device_id == other.device_id and
                self.mac_address == other.mac_address)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return hash((self.device_id, self.mac_address))

class XapiClient(object):
    SECURITY_GROUPS_KEY = "failmode"
    SECURITY_GROUPS_VALUE = "secure"

    def __init__(self):
        session = self._session()
        try:
            self._host_ref = session.xenapi.session.get_this_host(
                session.handle)
            self._host_uuid = session.xenapi.host.get_uuid(self._host_ref)
        finally:
            session.xenapi.logout()

    def _session(self):
        LOG.debug("Created new Xapi session")

        session = XenAPI.Session(CONF.AGENT.xapi_connection_url)
        session.login_with_password(CONF.AGENT.xapi_connection_username,
                                    CONF.AGENT.xapi_connection_password)
        return session

    def get_instances(self):
        """Returns a dict of `VM OpaqueRef` (str) -> `device_id` (str)."""
        LOG.debug("Getting instances from Xapi")

        session = self._session()
        try:
            recs = session.xenapi.VM.get_all_records()
        finally:
            session.xenapi.logout()

        # NOTE(asadoughi): Copied from xen-networking-scripts/utils.py
        is_inst = lambda r: (r['power_state'].lower() == 'running' and
                             not r['is_a_template'] and
                             not r['is_control_domain'] and
                             ('nova_uuid' in r['other_config'] or
                              r['name_label'].startswith('instance-')))
        instances = dict()
        for vm_ref, rec in recs.iteritems():
            if not is_inst(rec):
                continue
            instances[vm_ref] = rec["other_config"]["nova_uuid"]
        return instances

    def get_interfaces(self, instances):
        """Returns a set of VIFs from `get_instances` return value."""
        LOG.debug("Getting interfaces from Xapi")

        session = self._session()
        try:
            recs = session.xenapi.VIF.get_all_records()
        finally:
            session.xenapi.logout()

        interfaces = set()
        for rec in recs.values():
            device_id = instances.get(rec["VM"])
            if not device_id:
                continue
            interfaces.add(VIF(device_id, rec["MAC"]))
        return interfaces

    def _process_security_groups(self, session, vm_ref, vif, fn):
        """Process security groups setting on VM's Xenstore for VIF.

        Based on the argument function, VM Opaque Ref, and VIF, we set the
        appropriate VM's Xenstore data to enable security groups or disable
        security groups for the correct VIF.
        """
        vm = session.xenapi.VM.get_record(vm_ref)
        location = ('vm-data/networking/%s' %
                    vif.mac_address.replace(':', ''))
        xsdata = json.loads(vm["xenstore_data"][location])

        # Update Xenstore dict based on fn
        fn(xsdata)

        # Update param Xenstore
        session.xenapi.VM.remove_from_xenstore_data(
            vm_ref, location)
        session.xenapi.VM.add_to_xenstore_data(
            vm_ref, location, json.dumps(xsdata))

        # Update running VM's Xenstore
        args = dict(host_uuid=self._host_uuid,
                    path=location,
                    value=json.dumps(xsdata),
                    dom_id=session.xenapi.VM.get_domid(vm_ref))
        if not args["dom_id"] or args["dom_id"] == -1:
            # If the VM is not running, no need to update the live Xenstore
            return

        session.xenapi.host.call_plugin(
            self._host_ref, "xenstore.py", "write_record", args)

    def _set_security_groups(self, session, vm_refs, interfaces):
        LOG.debug("Setting security groups on %s", interfaces)

        for interface in interfaces:
            vm_ref = vm_refs[interface.device_id]
            self._process_security_groups(
                session,
                vm_ref,
                interface,
                lambda d: d.update({self.SECURITY_GROUPS_KEY:
                                    self.SECURITY_GROUPS_VALUE}))

    def _unset_security_groups(self, session, vm_refs, interfaces):
        LOG.debug("Unsetting security groups on %s", interfaces)

        for interface in interfaces:
            vm_ref = vm_refs[interface.device_id]
            self._process_security_groups(
                session,
                vm_ref,
                interface,
                lambda d: d.pop(self.SECURITY_GROUPS_KEY, None))

    def _refresh_interfaces(self, session, interfaces):
        LOG.debug("Refreshing devices on %s", interfaces)

        device_ids = set([vif.device_id for vif in interfaces])
        for device_id in device_ids:
            args = {"host_uuid": self._host_uuid, "uuid": device_id}
            session.xenapi.host.call_plugin(
                self._host_ref,
                "post_live_migrate",
                "instance_post_live_migration",
                args)

    def update_interfaces(self, instances, added_sg, updated_sg, removed_sg):
        """Handles changes to interfaces' security groups

        Calls refresh_interfaces on argument VIFs. Set security groups on
        added_sg's VIFs. Unsets security groups on removed_sg's VIFs.
        """
        if not (added_sg or updated_sg or removed_sg):
            return

        session = self._session()
        vm_refs = dict((v, k) for k, v in instances.iteritems())
        try:
            self._set_security_groups(session, vm_refs, added_sg)
            self._unset_security_groups(session, vm_refs, removed_sg)
            self._refresh_interfaces(session,
                                     added_sg + updated_sg + removed_sg)
        finally:
            session.xenapi.logout()


import filters
import cv2
import math
import sys
import numpy as np

class Hough(object):

    @staticmethod
    def apply(img, count_alpha, count_rad, threshold):
        img = filters.Filters().diff_image(img, 'laplace')
        cont_prep = Hough._get_countour_preparation(img, threshold)

        rad_max = math.sqrt(math.pow(img.shape[0], 2) + math.pow(img.shape[1], 2))
        delta_alpha = math.pi / (count_alpha - 1)
        delta_rad = 2.0 * rad_max / (count_rad - 1)
        hough_img = np.zeros((count_rad, count_alpha, 3), np.uint8)

        for pixel in cont_prep:
            alpha = -(math.pi / 2)
            for i in xrange(count_alpha):
                R = (pixel[0] * math.cos(alpha) + pixel[1] * math.sin(alpha) + rad_max) / delta_rad
                hough_img[int(R), i] += 1
                alpha += delta_alpha
        return hough_img

    @staticmethod
    def _get_countour_preparation(img, threshold):
        cont_prep = []
        for row in xrange(img.shape[0]):
            for col in xrange(img.shape[1]):
                color_sum = 0
                for channel in xrange(3):
                    color_sum += img[row, col, channel]

                intensity = color_sum / 3
                if intensity >= threshold:
                    cont_prep.append([row, col])
        return cont_prep

if __name__ == '__main__':
    img = cv2.imread(sys.argv[1])

    cv2.imshow('img', img)
    cv2.imshow('hough', Hough.apply(img, 800, 600, 240))
    if cv2.waitKey(0) & 0xFF == ord('s'):
        cv2.imwrite('hough.png', img)
    cv2.destroyAllWindows()


import numpy as np
from sklearn.cross_validation import StratifiedShuffleSplit
from numpy.random import random_sample
from collections import Counter

class SNN():
	"""
		A simple one class classifier, implemented for outlier removal.
		To be used with outlier removal function.
	"""

	def __init__(self, error_tolerance=10e-4, learning_rate = .05, outlier_removal_percent=.05):
		self.learning_rate = learning_rate
		self.error_tolerance = error_tolerance
		self.outlier_removal_percent = outlier_removal_percent

	def fit(self, X):

		#converts X to np array
		if (type(X) == list):
			X = np.array(X)

		#randomly initialize weights
		n_samples, n_features = X.shape
		self.w_ = random_sample((1 + n_features, ))
		self.e_ = []
		y = np.ones(n_samples)

		#regularize features from 0 to 1
		for feat in range(n_features):
			min_val = min(X[:, feat])
			max_min_range = max(X[:, feat]) - min_val
			for sample in range(n_samples):
				X[sample, feat] = ((X[sample, feat] - min_val) / max_min_range)

		#iterate untill it converges
		improvement, total_error = 1, None
		while improvement >= self.error_tolerance:
			errors = []
			for xi, target in zip(X, y):
				error = self.predict(xi) - target
				update = self.learning_rate * error
				self.w_[1:] -= update * xi
				self.w_[0] -= update
				errors.append(abs(error))
			self.e_.append(errors)

			#calculates improvement of prediction
			if total_error:
				improvement = (total_error - sum(errors))/total_error
			total_error = sum(errors)

	def predict(self, xi):
		return np.dot(xi, self.w_[1:]) + self.w_[0]

	def id_outliers(self):
		top_X = int(len(self.e_[-1]) * self.outlier_removal_percent)
		error_per_sample = zip(range(len(self.e_[-1])), self.e_[-1])
		return [v[0] for v in sorted(error_per_sample, key=lambda x:-x[1])[:top_X]]

def id_outliers(features, labels, outlier_removal_iterations = 3, outlier_removal_percent=.05, clf_attempts = 5):
	"""
		An outlier removal function.
		Takes 3 adjustable parameters:
			outlier_removal_iterations -> number of iterations it will perform its removal function. In each iteration, the outliers identified are removed before the next iteration is performed
			outlier_removal_percent -> the percentage of outliers that will be removed from X at each iteration
			clf_attempts -> how many times the classifier is run. The sample is only tagged as outlier if in all attempts the sample is flagged as an outlier
			#to fix: issue with indexing. need to return index based on the original list
	"""
	for _ in range(outlier_removal_iterations):
		outliers_counter = Counter()
		for _ in range(clf_attempts):
			clf = SNN(outlier_removal_percent=.05)
			clf.fit(features)
			for outlier in clf.id_outliers():
				outliers_counter[outlier] += 1
		outliers = [k for k,v in outliers_counter.items() if v == clf_attempts]
	return outliers

def remove_outliers(features, labels, outlier_removal_iterations = 3, outlier_removal_percent=.05, clf_attempts = 5):
	outliers = id_outliers(features, labels, outlier_removal_iterations, outlier_removal_percent, clf_attempts)
	features = [feat for idx, feat in enumerate(features) if idx not in outliers ]
	labels = [lab for idx, lab in enumerate(labels) if idx not in outliers ]
	return features, labels, outliers


import aiohttp
import asyncio

class MakeRequest():
    """make http request to our server"""

    def __init__(self, config):
        connector = aiohttp.TCPConnector(limit=50, force_close=True)
        self.host = config.host
        self.header = {
            "Accept": "application/json, text/plain, */*", "Connection": "keep-alive", "Accept-Encoding": "gzip, deflate, br", "User-Agent": "hacksec cli/"+config.version}
        self.session = aiohttp.ClientSession(
            trust_env=True, connector=connector)

    def runner(func):
        """runner"""
        def wrapper(*args, **kwargs):
            loop = asyncio.get_event_loop()
            response = loop.run_until_complete(func(*args, **kwargs))
            return response
        return wrapper

    def authenticate(self, access_token):
        """authenticate to our server"""
        self.header["Authorization"] = "Bearer " + access_token

    @runner
    async def close_session(self):
        """close session"""
        await self.session.close()

    @runner
    async def get(self, endpoint):
        """make get request"""
        async with self.session.get(self.host + endpoint, headers=self.header) as response:
            return await response.json(), response.status

    @runner
    async def post(self, endpoint, payload):
        """make post request"""
        async with self.session.post(self.host + endpoint, json=payload, headers=self.header) as response:
            return await response.json(), response.status

    @runner
    async def upload(self, endpoint, file):
        """make post request"""
        form = aiohttp.FormData()
        form.add_field('file', file)
        async with self.session.post(self.host + endpoint, data=form, headers=self.header) as response:
            return await response.json(), response.status


from django.shortcuts import render
from django.http import HttpResponseRedirect
from django.contrib.auth.decorators import permission_required
from shop.models import Item, List, UserModel, Store
from shop.forms import AddItemForm
from django.views import generic
from faker import Faker
from django.contrib.auth.forms import PasswordChangeForm

# Create your views here.

class homepage(generic.ListView):
    model = Store
    template_name = "homepage.html"

def store_items(request):
    store = Store.objects.filter(storename__exact="defaultStore")
    context = {"store" : store}
    return render(request, "store_items.html", context=context)

def profile(request):
    #user = UserModel.objects.filter(username__exact="defaultUser")
    #context = {"user": user}
    return render(request, "profilePage.html")

class map(generic.ListView):
    model = Store
    template_name = "map.html"

def signin(request):
    return render(request, "signin.html")

def favorite_cart(request):
    if request.user.is_authenticated:
        userM = UserModel.objects.filter(user__exact=request.user)
        user = request.user
        context = {"userM": userM,
                   "user": user}
        return render(request, "favorite_cart.html", context=context)
    return render(request, "login_error.html")

class StoreListView(generic.ListView):
    model = Store
    template_name = "store_list.html"

class StoreDetailView(generic.DetailView):
    model = Store
    template_name = "items_cart.html"

def favorite_items(request):
    if request.user.is_authenticated:
        userM = UserModel.objects.filter(user__exact=request.user)
        user = request.user
        context = {"userM": userM,
                   "user": user}
        return render(request, "favorite_items.html", context=context)
    return render(request, "login_error.html")

def favorite_items2(request):
    if request.user.is_authenticated:
        userM = UserModel.objects.filter(user__exact=request.user)
        user = request.user
        context = {"userM": userM,
                   "user": user}
        return render(request, "Stats2.html", context=context)
    return render(request, "login_error.html")

def user_preferences(request):
    if request.user.is_authenticated:
        userM = UserModel.objects.filter(user__exact=request.user)
        user = request.user
        context = {"userM": userM,
                   "user": user}

        if request.method == 'POST':
            form = PasswordChangeForm(request.user, request.POST)
            if form.is_valid():
                user = form.save()
                update_session_auth_hash(request, user)  # Important!
                messages.success(request, 'Your password was successfully updated!')
                return redirect('user_preferences')
            else:
                messages.error(request, 'Please correct the error below.')
        else:
            #form = PasswordChangeForm(request.user)
            form = PasswordChangeForm(user)
        return render(request, 'user_preferences.html', {
            'form': form
        })

        #return render(request, "user_preferences.html", context=context)
    return render(request, "login_error.html")

def login_error(request):
    return render(request, "login_error.html")

@permission_required('shop.can_add_items')
def addItem(request):
    fake = Faker()
    if request.user.is_authenticated:
        userM = UserModel.objects.filter(user__exact=request.user)
        user = request.user
        context = {"userM": userM,
                   "user": user}
        if request.method == 'POST':
            form = AddItemForm(request.POST)
            if form.is_valid():
                fakeStore = Store(name=fake.text(50), location=fake.text(10))
                fakeStore.save()
                i_name = form.cleaned_data['new_item']
                i_store = fakeStore
                i_price = fake.random_int(1, 100)
                i_aisle = fake.random_int(1, 26)
                i_short_description = fake.text(150)
                i_item_number = fake.random_int(1, 126)
                item = Item(name=i_name,
                            store=i_store,
                            price=i_price,
                            aisle=i_aisle,
                            short_description=i_short_description,
                            item_number=i_item_number,
                            )
                item.save()
                userM[0].favoriteItems.add(item)
                userM[0].save()
                return HttpResponseRedirect('success/')
        else:
            fakeStore = Store(name=fake.text(50), location=fake.text(10))
            fakeStore.save()
            i_name = fake.text(50)
            i_store = fakeStore
            i_price = fake.random_int(1, 100)
            i_aisle = fake.random_int(1, 26)
            i_short_description = fake.text(150)
            i_item_number = fake.random_int(1, 126)
            def_item = Item(name=i_name,
                        store=i_store,
                        price=i_price,
                        aisle=i_aisle,
                        short_description=i_short_description,
                        item_number=i_item_number,
                        )

            form = AddItemForm(initial={'new_item': def_item,})
        return render(request, "addItem.html", {'form': form})
    return render(request, "login_error.html")

def addItem_success(request):
    return render(request, "success.html")


# coding: utf-8

"""
    COVID-19 Local API

    API to get local help information about COVID-19 (hotlines, websites, test sites, health departments)  # noqa: E501

    The version of the OpenAPI document: 0.1.0
    Generated by: https://openapi-generator.tech
"""

import pprint
import re  # noqa: F401

import six

from covid_local.configuration import Configuration

class TestSite(object):
    """NOTE: This class is auto generated by OpenAPI Generator.
    Ref: https://openapi-generator.tech

    Do not edit the class manually.
    """

    """
    Attributes:
      openapi_types (dict): The key is attribute name
                            and the value is attribute type.
      attribute_map (dict): The key is attribute name
                            and the value is json key in definition.
    """
    openapi_types = {
        'country_code': 'str',
        'lat': 'float',
        'lon': 'float',
        'name': 'str',
        'street': 'str',
        'zip_code': 'int',
        'city': 'str',
        'address_supplement': 'str',
        'phone': 'str',
        'website': 'str',
        'operating_hours': 'str',
        'appointment_required': 'bool',
        'description': 'str',
        'sources': 'str',
        'distance': 'float'
    }

    attribute_map = {
        'country_code': 'country_code',
        'lat': 'lat',
        'lon': 'lon',
        'name': 'name',
        'street': 'street',
        'zip_code': 'zip_code',
        'city': 'city',
        'address_supplement': 'address_supplement',
        'phone': 'phone',
        'website': 'website',
        'operating_hours': 'operating_hours',
        'appointment_required': 'appointment_required',
        'description': 'description',
        'sources': 'sources',
        'distance': 'distance'
    }

    def __init__(self, country_code=None, lat=None, lon=None, name=None, street=None, zip_code=None, city=None, address_supplement=None, phone=None, website=None, operating_hours=None, appointment_required=None, description=None, sources=None, distance=None, local_vars_configuration=None):  # noqa: E501
        """TestSite - a model defined in OpenAPI"""  # noqa: E501
        if local_vars_configuration is None:
            local_vars_configuration = Configuration()
        self.local_vars_configuration = local_vars_configuration

        self._country_code = None
        self._lat = None
        self._lon = None
        self._name = None
        self._street = None
        self._zip_code = None
        self._city = None
        self._address_supplement = None
        self._phone = None
        self._website = None
        self._operating_hours = None
        self._appointment_required = None
        self._description = None
        self._sources = None
        self._distance = None
        self.discriminator = None

        if country_code is not None:
            self.country_code = country_code
        if lat is not None:
            self.lat = lat
        if lon is not None:
            self.lon = lon
        if name is not None:
            self.name = name
        if street is not None:
            self.street = street
        if zip_code is not None:
            self.zip_code = zip_code
        if city is not None:
            self.city = city
        if address_supplement is not None:
            self.address_supplement = address_supplement
        if phone is not None:
            self.phone = phone
        if website is not None:
            self.website = website
        if operating_hours is not None:
            self.operating_hours = operating_hours
        if appointment_required is not None:
            self.appointment_required = appointment_required
        if description is not None:
            self.description = description
        if sources is not None:
            self.sources = sources
        if distance is not None:
            self.distance = distance

    @property
    def country_code(self):
        """Gets the country_code of this TestSite.  # noqa: E501

        :return: The country_code of this TestSite.  # noqa: E501
        :rtype: str
        """
        return self._country_code

    @country_code.setter
    def country_code(self, country_code):
        """Sets the country_code of this TestSite.

        :param country_code: The country_code of this TestSite.  # noqa: E501
        :type: str
        """

        self._country_code = country_code

    @property
    def lat(self):
        """Gets the lat of this TestSite.  # noqa: E501

        :return: The lat of this TestSite.  # noqa: E501
        :rtype: float
        """
        return self._lat

    @lat.setter
    def lat(self, lat):
        """Sets the lat of this TestSite.

        :param lat: The lat of this TestSite.  # noqa: E501
        :type: float
        """

        self._lat = lat

    @property
    def lon(self):
        """Gets the lon of this TestSite.  # noqa: E501

        :return: The lon of this TestSite.  # noqa: E501
        :rtype: float
        """
        return self._lon

    @lon.setter
    def lon(self, lon):
        """Sets the lon of this TestSite.

        :param lon: The lon of this TestSite.  # noqa: E501
        :type: float
        """

        self._lon = lon

    @property
    def name(self):
        """Gets the name of this TestSite.  # noqa: E501

        :return: The name of this TestSite.  # noqa: E501
        :rtype: str
        """
        return self._name

    @name.setter
    def name(self, name):
        """Sets the name of this TestSite.

        :param name: The name of this TestSite.  # noqa: E501
        :type: str
        """

        self._name = name

    @property
    def street(self):
        """Gets the street of this TestSite.  # noqa: E501

        :return: The street of this TestSite.  # noqa: E501
        :rtype: str
        """
        return self._street

    @street.setter
    def street(self, street):
        """Sets the street of this TestSite.

        :param street: The street of this TestSite.  # noqa: E501
        :type: str
        """

        self._street = street

    @property
    def zip_code(self):
        """Gets the zip_code of this TestSite.  # noqa: E501

        :return: The zip_code of this TestSite.  # noqa: E501
        :rtype: int
        """
        return self._zip_code

    @zip_code.setter
    def zip_code(self, zip_code):
        """Sets the zip_code of this TestSite.

        :param zip_code: The zip_code of this TestSite.  # noqa: E501
        :type: int
        """

        self._zip_code = zip_code

    @property
    def city(self):
        """Gets the city of this TestSite.  # noqa: E501

        :return: The city of this TestSite.  # noqa: E501
        :rtype: str
        """
        return self._city

    @city.setter
    def city(self, city):
        """Sets the city of this TestSite.

        :param city: The city of this TestSite.  # noqa: E501
        :type: str
        """

        self._city = city

    @property
    def address_supplement(self):
        """Gets the address_supplement of this TestSite.  # noqa: E501

        :return: The address_supplement of this TestSite.  # noqa: E501
        :rtype: str
        """
        return self._address_supplement

    @address_supplement.setter
    def address_supplement(self, address_supplement):
        """Sets the address_supplement of this TestSite.

        :param address_supplement: The address_supplement of this TestSite.  # noqa: E501
        :type: str
        """

        self._address_supplement = address_supplement

    @property
    def phone(self):
        """Gets the phone of this TestSite.  # noqa: E501

        :return: The phone of this TestSite.  # noqa: E501
        :rtype: str
        """
        return self._phone

    @phone.setter
    def phone(self, phone):
        """Sets the phone of this TestSite.

        :param phone: The phone of this TestSite.  # noqa: E501
        :type: str
        """

        self._phone = phone

    @property
    def website(self):
        """Gets the website of this TestSite.  # noqa: E501

        :return: The website of this TestSite.  # noqa: E501
        :rtype: str
        """
        return self._website

    @website.setter
    def website(self, website):
        """Sets the website of this TestSite.

        :param website: The website of this TestSite.  # noqa: E501
        :type: str
        """

        self._website = website

    @property
    def operating_hours(self):
        """Gets the operating_hours of this TestSite.  # noqa: E501

        :return: The operating_hours of this TestSite.  # noqa: E501
        :rtype: str
        """
        return self._operating_hours

    @operating_hours.setter
    def operating_hours(self, operating_hours):
        """Sets the operating_hours of this TestSite.

        :param operating_hours: The operating_hours of this TestSite.  # noqa: E501
        :type: str
        """

        self._operating_hours = operating_hours

    @property
    def appointment_required(self):
        """Gets the appointment_required of this TestSite.  # noqa: E501

        :return: The appointment_required of this TestSite.  # noqa: E501
        :rtype: bool
        """
        return self._appointment_required

    @appointment_required.setter
    def appointment_required(self, appointment_required):
        """Sets the appointment_required of this TestSite.

        :param appointment_required: The appointment_required of this TestSite.  # noqa: E501
        :type: bool
        """

        self._appointment_required = appointment_required

    @property
    def description(self):
        """Gets the description of this TestSite.  # noqa: E501

        :return: The description of this TestSite.  # noqa: E501
        :rtype: str
        """
        return self._description

    @description.setter
    def description(self, description):
        """Sets the description of this TestSite.

        :param description: The description of this TestSite.  # noqa: E501
        :type: str
        """

        self._description = description

    @property
    def sources(self):
        """Gets the sources of this TestSite.  # noqa: E501

        :return: The sources of this TestSite.  # noqa: E501
        :rtype: str
        """
        return self._sources

    @sources.setter
    def sources(self, sources):
        """Sets the sources of this TestSite.

        :param sources: The sources of this TestSite.  # noqa: E501
        :type: str
        """

        self._sources = sources

    @property
    def distance(self):
        """Gets the distance of this TestSite.  # noqa: E501

        :return: The distance of this TestSite.  # noqa: E501
        :rtype: float
        """
        return self._distance

    @distance.setter
    def distance(self, distance):
        """Sets the distance of this TestSite.

        :param distance: The distance of this TestSite.  # noqa: E501
        :type: float
        """

        self._distance = distance

    def to_dict(self):
        """Returns the model properties as a dict"""
        result = {}

        for attr, _ in six.iteritems(self.openapi_types):
            value = getattr(self, attr)
            if isinstance(value, list):
                result[attr] = list(map(
                    lambda x: x.to_dict() if hasattr(x, "to_dict") else x,
                    value
                ))
            elif hasattr(value, "to_dict"):
                result[attr] = value.to_dict()
            elif isinstance(value, dict):
                result[attr] = dict(map(
                    lambda item: (item[0], item[1].to_dict())
                    if hasattr(item[1], "to_dict") else item,
                    value.items()
                ))
            else:
                result[attr] = value

        return result

    def to_str(self):
        """Returns the string representation of the model"""
        return pprint.pformat(self.to_dict())

    def __repr__(self):
        """For `print` and `pprint`"""
        return self.to_str()

    def __eq__(self, other):
        """Returns true if both objects are equal"""
        if not isinstance(other, TestSite):
            return False

        return self.to_dict() == other.to_dict()

    def __ne__(self, other):
        """Returns true if both objects are not equal"""
        if not isinstance(other, TestSite):
            return True

        return self.to_dict() != other.to_dict()


import re
from django.conf import settings
from django.http import Http404, HttpResponseServerError
from django.utils import timezone
from datetime import timedelta, date,datetime
import datetime
import time
from django.utils.timezone import utc

from io import BytesIO
from django.http import HttpResponse
from django.template.loader import get_template

from xhtml2pdf import pisa

from rest_framework.response import Response

from portalbackend.lendapi import constants
from portalbackend.lendapi.accounting.models import AccountingOauth2, CoA,LoginInfo
from portalbackend.lendapi.accounting.models import FinancialStatementEntryTag, TrialBalance, CoAMap
from portalbackend.lendapi.reporting.models import FinancialStatementEntry
from django.db import transaction
import calendar

class AccountingUtils(object):
    """
    A collection of functions that are used within the Accounting views and functions
    """
    @staticmethod
    def updateAccountingSession(company, accessToken, refreshToken, realmId):
        """
        Updates the Accounting Session Saved inside the database
        :param company:
        :param accessToken:
        :param refreshToken:
        :param realmID:
        :return:
        """
        token = AccountingOauth2.objects.filter(company=company).first()
        if token:
            token.accessToken = accessToken
            token.refreshToken = refreshToken
            if token.realmId != realmId:
                token.realmId = realmId
        else:
            token = AccountingOauth2(company=company, accessToken=accessToken,
                                     refreshToken=refreshToken, realmId=realmId)
        token.save()
        return token

    @staticmethod
    def get_credentials(request):
        """
        Fetches the accounting authentication credentials by the request user (deprecated)
        :param request:
        :return:
        """
        if request.user.company:
            token = AccountingOauth2.objects.filter(company=request.user.company).first()
            if token:
                return token
            raise Http404
        else:
            raise Http404

    @staticmethod
    def get_status_object(company):
        now = datetime.datetime.utcnow().replace(tzinfo=utc)
        time_threshold = now - datetime.timedelta(hours=2)
        try:
            return LoginInfo.objects.filter(company=company, created__range=(time_threshold, now)).first()
        except LoginInfo.DoesNotExist:
            raise Http404

    @staticmethod
    def get_credentials_by_company(company):
        """
        Fetches the accounting authentication credentials by company object
        :param company:
        :return:
        """
        token = AccountingOauth2.objects.filter(company=company).first()
        return token

    @staticmethod
    def create_mapping_dict(queryset):
        """
        Converts a queryset of the default map into a useable dictionary
        :param queryset:
        :return:
        """
        d = {}
        for mapping in queryset:
            mapping_key = re.sub ('[^a-zA-Z0-9]', '', mapping.account_category.lower())
            d[mapping_key] = {"default_map_id": mapping.default_map_id,
                                                   "default_map_name": mapping.default_map_name}
        return d

    @staticmethod
    def parse_statement(request,company, data, statement_type):
        """
        Parses the income statement and balance sheet sheet request, sent in the form of:
         {"StatementType": [{"Period": "2016-01"},...]}, statement type can either be BalanceSheet or IncomeStatement
        :param company:
        :param data:
        :param statement_type: The type of sheet that is being parsed
        :return:
        """
        fixed_fields = ("Period", "CreatedTimestamp", "LastUpdatedTimestamp", "SourceName", "SourceKey")
        entries = []
        error_tags = []
        monthly_report = None
        tags = FinancialStatementEntryTag.objects.all()

        tag_lookup = {}
        sight_tag_lookup = {}
        for tag in tags:
            sight_tag_lookup[tag.all_sight_name] = tag.tag_id
            tag_lookup[str(tag.tag_id)] = {"id": tag.id, "short_label": tag.short_label}

        # All Sight sends back BalanceSheet not Balance Sheet, so we need to translate (same for income statement)
        for month in data[statement_type.replace(" ", "")]:
            y, m = map(int, month["Period"].split('-'))
            num_days = calendar.monthrange(y, m)[1]
            period = datetime.date(y, m, num_days)

            for key, value in month.items():
                if value is None:
                    value = '0'

                if key not in fixed_fields:
                    check_key_exists = sight_tag_lookup.get (key, None)
                    if check_key_exists is None:
                        tag_key = str(key)
                    else:
                        tag_key = sight_tag_lookup[key]

                    try:

                        cur_tag = tag_lookup[str(tag_key)]
                    except KeyError as e:
                        print('### WARNING ### - tag id ', tag_key, ' not found in financial statement entry tag map')
                        if tag_key not in error_tags:
                            error_tags.append(tag_key)

                    else:
                        rows_affected = FinancialStatementEntry.objects.filter(
                                            company=company,
                                            period_ending=period,
                                            statement_type=statement_type,
                                            fse_tag_id=cur_tag['id']).update(value=value)

                        # 0 rows affected == it doesn't exist
                        if rows_affected == 0:
                            # these will be bulk created
                            entry = FinancialStatementEntry(company=company, period_ending=period,
                                                            entry_name=cur_tag['short_label'],
                                                            value=value, currency='CAD',
                                                            fse_tag_id=cur_tag['id'],
                                                            statement_type=statement_type)
                            entries.append(entry)
        return entries,error_tags

    @staticmethod
    def credit_debit_mismatch(company):
        """
        Method compares the Total Assets and Total L&E and returns the period if both the value are not equal
        """
        try:
            total_liabilities_equity = FinancialStatementEntryTag.objects.get(
                tag_category="total_liabilities_equity")
            total_assets = FinancialStatementEntryTag.objects.get(tag_category="total_assets")
            financial_statement_entry_equity = FinancialStatementEntry.objects.filter(
                company=company,
                statement_type=FinancialStatementEntry.BALANCE_SHEET,
                fse_tag_id=total_liabilities_equity)
            dict_credits = {}
            dict_debits = {}
            for financial_statement_entry_object in financial_statement_entry_equity:
                dict_credits[str(financial_statement_entry_object.period_ending)] = int(
                    financial_statement_entry_object.value)
            financial_statement_entry_assets = FinancialStatementEntry.objects.filter(
                company=company,
                statement_type=FinancialStatementEntry.BALANCE_SHEET,
                fse_tag_id=total_assets)
            for financial_statement_entry_object in financial_statement_entry_assets:
                dict_debits[str(financial_statement_entry_object.period_ending)] = int(
                    financial_statement_entry_object.value)
            credit_debit_mismatch = []
            for key in dict_credits:
                if key in dict_debits:
                    val1 = dict_credits[key]
                    val2 = dict_debits[key]
                    if val1 != val2:
                        credit_debit_mismatch.append(key)
            return credit_debit_mismatch
        except Exception as e:
            print(str(e))
            return None

    @staticmethod
    def get_stripped_tags():
        """
        Retrieves stripped down version of Financial Statement Entry tags
        :return:
        """
        d = {}
        # some tags are formatted differently in the post than in the structure
        special_cases = {'TotalLE': 'TotalLiabilityAndEquity',
                         'EquityPortionofLTDebt': 'EquityPositionOfLTDebt',
                         'ShareCapitalandContributedCapital': 'ShareAndContributedCapital',
                         'NetIncome': 'NetIncomeYTD',
                         'SREDITCReceivable': 'SREDReceivable',
                         'PatentsIntangibleAssets': 'PatentsAndIntangibleAssets',
                         'TotalRevenues': 'TotalRevenue',
                         'DepreciationAmortization': 'DepreciationAndAmortization',
                         'SRED': 'SREDAccrual',
                         'RDGrossexcludingSRED': 'RDGrossMinusExcludingSRED',
                         'AccountsReceivables': 'AccountReceivables'
                         }
        tags = FinancialStatementEntryTag.objects.all()
        for tag in tags:
            stripped = tag.short_label
            stripped = stripped.replace(' ', '')
            repl = re.sub('[^a-zA-Z0-9 \n\.]', '', stripped)
            if repl in special_cases:
                repl = special_cases[repl]
            d[repl] = tag
        ld = {k.lower(): v for k, v in d.items()}
        return ld

    @staticmethod
    def build_all_sight_save_request(company):
        data = {"SourceName": "CustomerProfile",
                "SourceKey": company.id,
                "Model": {
                    "Financials": {}
                    }
                }
        trial_balances = TrialBalance.objects.filter(company=company).order_by('period')
        grouped = dict()
        for obj in trial_balances:
            grouped.setdefault(obj.period.strftime("%Y-%m"), []).append(obj)

        data["Model"]["Financials"]["CustomerTrialBalance"] = []
        for key, month_data in grouped.items():
            subdata = {"Period": key, "RecordedTimestamp": timezone.now().strftime("%Y-%m-%d %H:%M:%S.%f"),
                       "Version": "1", "CustomerTrialBalanceItem": []}

            for entry in month_data:
                subdata["CustomerTrialBalanceItem"].append({
                    "AccountId": str(entry.gl_account_id),
                    "AccountName": entry.gl_account_name,
                    "CreditAmount": str(entry.credit),
                    "DebitAmount": str(entry.debit)
                })
            data["Model"]["Financials"]["CustomerTrialBalance"].append(subdata)

        data["Model"]["Financials"]["CustomerAccount"] = []
        coas = CoA.objects.filter(company=company)
        for coa in coas:
            subdata = {
                "AccountId": str(coa.gl_account_id),
                "AccountName": coa.gl_account_name,
                "AccountCategory": coa.gl_account_type
            }
            data["Model"]["Financials"]["CustomerAccount"].append(subdata)

        data["Model"]["Financials"]["CustomerAccountMapping"] = []
        coamaps = CoAMap.objects.filter(company=company)
        for mapping in coamaps:
            subdata = {
                "FromAccountId": str(mapping.cust_account_id),
                "FromAccountName": mapping.cust_account_name,
                "ToAccountId": str(mapping.espresso_account_id),
                "ToAccountName": mapping.espresso_account_name,
            }
            data["Model"]["Financials"]["CustomerAccountMapping"].append(subdata)
        return data

    @staticmethod
    def create_coa_map(request,company, default_mappings, coa, remap):
        mappings = AccountingUtils.create_mapping_dict(default_mappings)
        saved_mappings = []

        for account in coa:
            try:
                # since we're forced to match on strings, we strip out all non-numeric / non-alpha characters
                # and lower all alpha to reduce the chance for mis match between submitted and default tag names
                lookup_key = re.sub('[^a-zA-Z0-9]', '', account.gl_account_type).lower()
                default_map_dict = mappings[lookup_key]
            except KeyError:
                # error = ["%s" % e]
                # return Utils.dispatch_failure(request, 'DATA_PARSING_ISSUE', error)
                # todo: log this to a file, and send a warning to the system admin to check
                print({"error": "no mapping for account type: {} ".format(account.gl_account_type)})
            else:
                entry = CoAMap.objects.filter(company=company, cust_account_id=account.gl_account_id).first()
                # send back anything that's new, so it can be mapped
                if not entry:
                    mapping = CoAMap(company=company,
                                     cust_account_name=account.gl_account_name,
                                     cust_account_id=account.gl_account_id,
                                     espresso_account_id=default_map_dict["default_map_id"],
                                     espresso_account_name=default_map_dict["default_map_name"])
                    mapping.save()
                    saved_mappings.append(mapping)
                else:
                    # and also send back anything that hasn't been verified by the user
                    # verification doesn't occur until the monthly report is signed off, because the
                    # remapping exercise done by the UI may need to take place several times before it can be considered
                    # valid

                    # Below logic for update changes in CoAMap
                    mapping_data = CoAMap.objects.get(id=entry.id)
                    if account.gl_account_name != entry.cust_account_name:
                        mapping_data.cust_account_name = account.gl_account_name
                        mapping_data.save()
                        saved_mappings.append (mapping_data)

                    if not entry.verified_by_user:
                        saved_mappings.append(entry)

        return saved_mappings

    @staticmethod
    def set_coa_map(company, coamap):
        updated_mappings = []
        for account in coamap:
            coamapping = CoAMap.objects.filter(company=company, cust_account_id=account["cust_account_id"]).first()

            tag = FinancialStatementEntryTag.objects.filter(description=account["espresso_account_name"],
                                                            tag_id=account["espresso_account_id"]).first()

            if not tag:
                tag = FinancialStatementEntryTag.objects.filter(tag_id=account["espresso_account_id"]).first()
            coamapping.espresso_account_id = tag.tag_id
            coamapping.espresso_account_name = tag.description
            coamapping.is_mapped = True
            coamapping.save()

            updated_mappings.append(coamapping)

        return updated_mappings

    @staticmethod
    def get_gl_account_id_by_name(company, account_name):
        gl_id = ''

        coa = CoA.objects.filter(company=company, gl_account_name=account_name).first()
        if coa:
            gl_id = coa.gl_account_id

        return gl_id

    @staticmethod
    def render_to_pdf(template_src, context_dict={}):
        template = get_template (template_src)
        html = template.render (context_dict)
        result = BytesIO ()
        pdf = pisa.pisaDocument (BytesIO (html.encode ("UTF-8")), result)
        if not pdf.err:
            return HttpResponse (result.getvalue (), content_type='application/pdf')
        return None


from typing import Callable, List, Literal, Optional, Sequence, TypeVar, Union
from pathlib import Path

from requests import HTTPError

import calcbench as cb
import pandas as pd
from tqdm.auto import tqdm

cb.enable_backoff(
    giveup=lambda e: isinstance(e, HTTPError) and (e.response.status_code in [404, 500])
)

T = TypeVar("T")

def iterate_to_dataframe(
    arguments: Sequence[T],
    f: Callable[[T], pd.DataFrame],
) -> pd.DataFrame:
    """Apply arguments to a function that returns a DataFrame append to a dataframe and return.

    Usage::

    >>> %pip install calcbench-api-client[Pandas,Backoff,tqdm]
    >>> from calcbench.downloaders import iterate_to_dataframe
    >>> import calcbench as cb
    >>> tickers = cb.tickers(entire_universe=True)
    >>> d = iterate_and_save_pandas(
    >>>    tickers,
    >>>    lambda ticker: cb.point_in_time(
    >>>        all_face=True,
    >>>        all_footnotes=False,
    >>>        company_identifiers=[ticker],
    >>>        all_history=True,
    >>>        include_preliminary=True,
    >>>        include_xbrl=True,
    >>>    ),
    >>> )
    """
    argument: T
    chunks = []
    for argument in tqdm(list(arguments)):
        try:
            d = f(argument)
            if d.empty:
                continue
        except KeyboardInterrupt:
            raise
        except Exception as e:
            tqdm.write(f"Exception getting {argument} {e}")
        else:
            chunks.append(d)
    if chunks:
        return pd.concat(chunks)
    else:
        return pd.DataFrame()

def iterate_and_save_pandas(
    arguments: Sequence[T],
    f: Callable[[T], pd.DataFrame],
    file_name: Union[str, Path],
    write_index: bool = True,
    columns: Optional[Sequence[str]] = None,
    write_mode: Literal["w", "a"] = "w",
):
    """Apply arguments to a function that returns a DataFrame and save to a .csv file.

    :param arguments: Each item in this sequence will be passed to f
    :param f: Function that generates a pandas dataframe that will be called on arguments
    :param file_name: Name of the file to write
    :param write_index: Write the pandas index to the csv file
    :param columns: which columns to write.  If this is set the index is not written
    :param write_mode: set the initial write mode.  "a" to append, "w" to overwrite.  Useful for resuming downloading.

    Usage::

    >>> %pip install calcbench-api-client[Pandas,Backoff,tqdm]
    >>> from calcbench.downloaders import iterate_and_save_pandas
    >>> import calcbench as cb
    >>> tickers = cb.tickers(entire_universe=True)
    >>> iterate_and_save_pandas(
    >>>    arguments=tickers,
    >>>    f=lambda ticker: cb.standardized(company_identifiers=[ticker], point_in_time=True),
    >>>    file_name="calcbench_standardized_PIT.csv",
    >>> )

    """
    argument: T
    write_headers = write_mode != "a"
    for argument in tqdm(list(arguments)):
        try:
            data = f(argument)
            if data.empty:
                continue
        except KeyboardInterrupt:
            raise
        except Exception as e:
            tqdm.write(f"Exception getting {argument} {e}")
        else:
            if columns:
                try:
                    data = data.reset_index()[columns]
                except Exception as e:
                    tqdm.write(f"Exception getting columns for {argument} {e}")
                    continue
            data.to_csv(
                file_name,
                mode=write_mode,
                index=write_index,
                header=write_headers,
            )
            write_mode = "a"
            write_headers = False

def iterate_and_save_pyarrow_dataset(
    arguments: Sequence[T],
    f: Callable[[T], pd.DataFrame],
    root_path: Union[str, Path],
    partition_cols: Optional[List[str]] = ["ticker"],
):
    """
    Apply the arguments to a function a save to a pyarrow dataset.

    Usage::

    >>> %pip install calcbench-api-client[Pandas,Backoff,tqdm,pyarrow]
    >>> tickers = sorted(cb.tickers(entire_universe=True), key=lambda ticker: hash(ticker)) # randomize the order so the time estimate is better
    >>> iterate_and_save_pyarrow_dataset(
    >>>     arguments=tickers,
    >>>     f=lambda ticker: cb.standardized(company_identifiers=[ticker], point_in_time=True),
    >>>     root_path="~/standardized_PIT_arrow/",
    >>>     partition_cols=["ticker"],
    >>> )
    >>> # Read the dataset
    >>> import pyarrow.parquet as pq
    >>> import pyarrow.compute as pc
    >>> table = pq.read_table(<root_path>)
    >>> expr = pc.field("ticker") == "MSFT"
    >>> msft_data = table.filter(expr).to_pandas()
    >>>
    >>> # Write the data
    >>> pq.write_table(table, "C:/Users/andre/Downloads/standardized_data.parquet")
    >>> from pyarrow import csv
    >>> csv.write_csv(table, "C:/Users/andre/Downloads/entire_universe_standardized_PIT.csv")

    """
    import pyarrow as pa
    import pyarrow.parquet as pq

    for argument in tqdm(list(arguments)):
        try:
            df = f(argument)
            if df.empty:
                continue
        except KeyboardInterrupt:
            raise
        except Exception as e:
            tqdm.write(f"Exception getting {argument} {e}")
        else:
            table = pa.Table.from_pandas(df)
            pq.write_to_dataset(
                table=table,
                root_path=root_path,
                partition_cols=partition_cols,
                **{"allow_truncated_timestamps": True, "coerce_timestamps": "us"},
            )


from typing import Any, Dict, List, TypedDict, cast

import fastjsonschema

from shared.di import injector
from shared.flask_frontend import InvalidRequest
from shared.typing import JSON
from shared.util import BadCodingError
from writer.core import (
    BaseRequestEvent,
    RequestCreateEvent,
    RequestDeleteEvent,
    RequestRestoreEvent,
    RequestUpdateEvent,
    Writer,
    WriteRequest,
)

write_schema = fastjsonschema.compile(
    {
        "$schema": "http://json-schema.org/draft-07/schema#",
        "type": "object",
        "properties": {
            "user_id": {"type": "integer"},
            "information": {},
            "locked_fields": {
                "type": "object",
                "additionalProperties": {"type": "integer"},
            },
            "events": {
                "type": "array",
                "items": {
                    "type": "object",
                    "properties": {
                        "type": {
                            "type": "string",
                            "enum": ["create", "update", "delete", "restore"],
                        },
                        "fqid": {"type": "string"},
                    },
                    "required": ["type", "fqid"],
                },
            },
        },
        "required": ["user_id", "information", "locked_fields", "events"],
    }
)

class WriteRequestJSON(TypedDict):
    user_id: int
    information: JSON
    locked_fields: Dict[str, int]
    events: List[Dict[str, Any]]

class WriteHandler:
    def write(self, data: JSON) -> None:
        write_request = self.build_write_request(data)
        writer = injector.get(Writer)
        writer.write(write_request)

    def build_write_request(self, data: JSON) -> WriteRequest:
        try:
            parsed_data = cast(WriteRequestJSON, write_schema(data))
        except fastjsonschema.JsonSchemaException as e:
            raise InvalidRequest(e.message)

        user_id = parsed_data["user_id"]
        information = parsed_data["information"]
        locked_fields = parsed_data["locked_fields"]
        events = self.parse_events(parsed_data["events"])

        return WriteRequest(events, information, user_id, locked_fields)

    def parse_events(self, events: List[Dict[str, Any]]) -> List[BaseRequestEvent]:
        request_events = []
        for event in events:
            type = event["type"]

            if type in ("create", "update"):
                fields = event.get("fields")
                if not isinstance(fields, dict):
                    raise InvalidRequest("Fields must be a dict")
                for key, value in fields.items():
                    if not isinstance(key, str):
                        raise InvalidRequest("Each key of fields must be a string")
            request_events.append(self.create_event(event))
        return request_events

    def create_event(self, event: Dict[str, Any]) -> BaseRequestEvent:
        type = event["type"]
        fqid = event["fqid"]
        request_event: BaseRequestEvent
        if type == "create":
            request_event = RequestCreateEvent(fqid, event["fields"])
        elif type == "update":
            request_event = RequestUpdateEvent(fqid, event["fields"])
        elif type == "delete":
            request_event = RequestDeleteEvent(fqid)
        elif type == "restore":
            request_event = RequestRestoreEvent(fqid)
        else:
            raise BadCodingError()
        return request_event

get_ids_schema = fastjsonschema.compile(
    {
        "$schema": "http://json-schema.org/draft-07/schema#",
        "type": "object",
        "properties": {
            "amount": {"type": "integer"},
            "collection": {"type": "string"},
        },
        "required": ["amount", "collection"],
    }
)

class GetIdsRequestJSON(TypedDict):
    collection: str
    amount: int

class GetIdsHandler:
    def get_ids(self, data: JSON) -> List[int]:
        try:
            parsed_data = cast(GetIdsRequestJSON, get_ids_schema(data))
        except fastjsonschema.JsonSchemaException as e:
            raise InvalidRequest(e.message)

        writer = injector.get(Writer)
        return writer.get_ids(parsed_data["collection"], parsed_data["amount"])


import cv2 as cv
import numpy as np
from PIL import Image

face_cascade = cv.CascadeClassifier('picture_processor/haarcascade_frontalface_alt.xml')

'''
Function to find any faces in an image
:param im: Image with faces
:return: location of the first face that is found or None if no face in the picture
'''
def find_face(im):
    image = np.array(im)
    image = cv.cvtColor(image, cv.COLOR_BGR2GRAY)
    faces = face_cascade.detectMultiScale(image)

    if isinstance(faces, tuple):
        return None

    return faces[0]

'''
Function to process and center image on a face
:param im: Image
:return: Processed image centered on the face or None if no face is found
'''
def process(im):
    face = find_face(im)

    if not isinstance(face, np.ndarray):
        return None

    #Center image on face
    dim = max(face[2],face[3])*2
    new = Image.new('RGB', (dim,dim), (255, 255, 255))
    new.save("test2.png")
    box = [int(face[0]-face[2]*0.5),int(face[1]-face[3]*0.5),int(face[0]+1.5*face[2]), int(face[1]+1.5*face[3])]

    im = im.crop(tuple(box))
    new.paste(im, (0,0,im.size[0],im.size[1]))
    new.thumbnail((512,512), Image.ANTIALIAS)

    return new


from Cookie import SimpleCookie

def make_set_cookie_header(name, value, path='/'):
    """
    Makes a 'Set-Cookie' header.

    """
    c = SimpleCookie()
    c[name] = value
    c[name]['path'] = path

    # can also set expires and other stuff.  See
    # Examples under http://docs.python.org/library/cookie.html.

    s = c.output()
    (key, value) = s.split(': ')
    return (key, value)

def cookie_expiration_date(numdays):
    # FROM http://python.6.n6.nabble.com/How-to-Delete-a-Cookie-td1528033.html
    """
    Returns a cookie expiration date in the required format.
    `expires` should be a string in the format "Wdy, DD-Mon-YY HH:MM:SS GMT"
    NOTE!  Must use [expires] because earlier IE versions don't support [max-age].
    A negative input value will delete the cookie
    """
    from datetime import date, timedelta
    new = date.today() + timedelta(days = numdays)
    return new.strftime("%a, %d-%b-%Y 23:59:59 GMT")


from django import forms
from category.models import Event, Category

class EventForm(forms.ModelForm):
    class Meta:
        model = Event
        fields = ['name', 'description', 'hold_place', 'image', 'pub_date', 'hold_date', 'category']
        widgets = {
            'name': forms.TextInput(),
            'description': forms.Textarea(),
            'hold_place': forms.TextInput(),
            'pub_date': forms.DateInput(format='%d/%m/%Y'),
            'hold_date': forms.TextInput(),
        }
        categories = Category.objects.all()
        choices = []
        for cat in categories:
            choices.append(tuple(cat.name))

        category = forms.ChoiceField(choices=tuple(choices))
        image = forms.ImageField(max_length=500, allow_empty_file=True, required=False)


import sys # cache disable
sys.dont_write_bytecode = True

from pynput.keyboard import Key, Listener # $ pip3 install pynput
import orta_ttyin, os

cmd_pressed = False # is command pressed? initial value

def toggle_cmd_pressed(): # cmd DOWN -> true, cmd UP -> false
    global cmd_pressed
    cmd_pressed = not cmd_pressed

def on_press(key):
    if len(str(key)) > 3: # dismisses 1-letter key e.g. = 'p'
        msg = ('{0} DOWN'.format(key)).replace("Key.","") # format = button UP
        print("keyevent", msg)
        #orta_ttyin.pushCommand("keyevent"+msg) #send to ortaOS
    if key in {Key.cmd,Key.cmd_l,Key.cmd_r}:
        toggle_cmd_pressed()
    elif cmd_pressed == True:
        if str(key) in {"'c'","'x'","'v'"}:
            print("commandkeyevent Copy&Paste")
            #orta_ttyin.pushCommand("commandkeyevent Copy&Paste")
        elif str(key) in {"'w'","'q'"}:
            print("commandkeyevent ForceQuit")
            #orta_ttyin.pushCommand("commandkeyevent ForceQuit")

def on_release(key):
    if key in {Key.cmd,Key.cmd_l,Key.cmd_r}: # only sends cmd UP
        toggle_cmd_pressed()
        msg = ('{0} UP'.format(key)).replace("Key.","")
        print("keyevent",msg)
        #orta_ttyin.pushCommand("keyevent"+msg)
    if key == Key.esc:
        """
        purpose: Stopping listener.
        change condition to

        not os.path.exists('/tmp/HARTS/ortaos/vrootfs/System')

        for actual use.
        """
        return False

# Collect events until on_release returns False.
with Listener(on_press=on_press,on_release=on_release) as listener:
    listener.join()


from django.shortcuts import render
from django.shortcuts import render, redirect
from django.contrib.auth import authenticate, login
from django.contrib import auth
from django.views.generic import View
from django.template import loader
from django.http import HttpResponse
from django.core.urlresolvers import reverse
# Create your views here.
import mysql.connector as pysql
import subprocess as sb
import sys,os,time

def index (request):
	client = request.session.get('client')
	print (client)
	no_cluster = request.session.get('no_cluster')
	print(no_cluster)
	if (no_cluster=='No Cluster'):
		return render (request,'client/client.html',{'no_cluster':no_cluster})
	elif (client==None):
		cluster_existed = 'cluster_existed'
		return render (request,'client/client.html',{'cluster_existed':cluster_existed})
	elif (client=='created'):
		return redirect ('/client/home/')

def existing (request):
	request.session['client_name'] = 'container_2'
	request.session['client_id'] = sb.getoutput('sudo docker exec container_2 hostname')
	request.session['client_ip'] = sb.getoutput('sudo docker exec container_2 hostname -i')
	request.session['client'] = 'created'
	return redirect ('/client/home/')

def new (request):
	os.system (' sudo docker run -itd --privileged --restart always --name client hadoopv1.2')
	os.system (' sudo docker cp /root/docker_hv1/core-site.xml client:/etc/hadoop/core-site.xml')
	request.session['client_name'] = 'client'
	request.session['client_id'] = sb.getoutput('sudo docker exec client hostname')
	request.session['client_ip'] = sb.getoutput('sudo docker exec client hostname -i')
	request.session['client'] = 'created'
	return redirect ('/client/home/')

def home (request):
	client_name = request.session.get('client_name')
	client_ip = request.session.get('client_ip')
	client_id = request.session.get('client_id')
	command = request.session.get('command')
	request.session['command'] = None
	return render (request,'client/home.html',{
		'client_name':client_name,
		'client_ip':client_ip,
		'client_id':client_id,
		'command':command           })

def del_client (request):
	os.system('docker kill client')
	os.system('docker rm client')
	request.session['client_name'] = None
	request.session['client_id'] = None
	request.session['client_ip'] = None
	request.session['client'] = None
	return redirect ('/client/')

def ls (request):
	client_name = request.session.get('client_name')
	command = sb.getoutput('docker exec '+client_name+' hadoop fs -ls /')
	request.session['command'] = command
	return redirect ('/client/')

def cat (request):
	location = request.POST.get('location')
	client_name = request.session.get('client_name')
	command = sb.getoutput('docker exec '+client_name+' hadoop fs -cat '+location)
	request.session['command'] = command
	return redirect ('/client/')

def mkdir (request):
	location = request.POST.get('location')
	client_name = request.session.get('client_name')
	sb.getoutput('docker exec '+client_name+' hadoop fs -mkdir '+location)
	return redirect ('/client/')

def rmr (request):
	location = request.POST.get('location')
	client_name = request.session.get('client_name')
	sb.getoutput('docker exec '+client_name+' hadoop fs -rmr '+location)
	return redirect ('/client/')

def touchz (request):
	location = request.POST.get('location')
	client_name = request.session.get('client_name')
	sb.getoutput('docker exec '+client_name+' hadoop fs -touchz '+location)
	return redirect ('/client/')

def rm (request):
	location = request.POST.get('location')
	client_name = request.session.get('client_name')
	sb.getoutput('docker exec '+client_name+' hadoop fs -rm '+location)
	return redirect ('/client/')

def clear_all (request):
	client_name = request.session.get('client_name')
	all_files = sb.getoutput('docker exec '+ client_name + ' hadoop fs -ls / | cut -d "/" -f2 | grep -vi Found')
	directories = all_files.split()
	print(directories)
	for files in directories:
		print ('cleaning /'+files+' ...')
		sb.getoutput ('docker exec '+ client_name + ' hadoop fs -rmr /'+files)
	return redirect ('/client/')

def put (request):
	location_from = request.POST.get('location_from')
	location_to = request.POST.get('location_to')
	client_name = request.session.get('client_name')
	sb.getoutput('docker exec '+client_name+' hadoop fs -put '+location_from+ ' '+location_to)
	return redirect ('/client/')

def moveFromLocal (request):
	location_from = request.POST.get('location_from')
	location_to = request.POST.get('location_to')
	client_name = request.session.get('client_name')
	sb.getoutput('docker exec '+client_name+' hadoop fs -moveFromLocal '+location_from+' '+location_to)
	return redirect ('/client/')

def chown (request):
	user = request.POST.get('user')
	location = request.POST.get('location')
	client_name = request.session.get('client_name')
	sb.getoutput('docker exec '+client_name+' hadoop fs -chown '+user+ ' '+location)
	return redirect ('/client/')

def count (request):
	location = request.POST.get('location')
	client_name = request.session.get('client_name')
	command = sb.getoutput('docker exec '+client_name+' hadoop fs -count '+location)
	request.session['command'] = command
	return redirect ('/client/')


#!/usr/bin/env python
import sys, argparse, boto3, glob, socket, posixpath as pp
import botocore.exceptions
import deploy

parser = argparse.ArgumentParser(description='Set up localstack environment.')
parser.add_argument('code_path', help='Path to Lambda code zip file')
arguments = parser.parse_args()

# Names of things

host_address = socket.gethostbyname(socket.gethostname())

BUCKETNAME = 'planscore'
ENDPOINT_S3 = 'http://{}:4572'.format(host_address)
ENDPOINT_LAM = 'http://{}:4574'.format(host_address)
AWS_CREDS = dict(aws_access_key_id='nobody', aws_secret_access_key='nothing')
CODE_PATH = arguments.code_path

# S3 Bucket setup

print('--> Set up S3', ENDPOINT_S3)
s3 = boto3.client('s3', endpoint_url=ENDPOINT_S3, **AWS_CREDS)

print('    Create bucket', BUCKETNAME)
s3.create_bucket(Bucket=BUCKETNAME)

def upload(prefix, basedir, pattern):
    for path in glob.glob(pattern):
        with open(path, 'rb') as file:
            data = file.read()

        key = pp.join(prefix, pp.relpath(path, basedir))

        s3.put_object(Bucket=BUCKETNAME, Key=key, ACL='public-read',
            Body=data, ContentType='text/json')

        print('    Put object', key, 'from', file.name)

prefix1 = pp.join('data', 'XX', '001')
basedir1 = pp.join(pp.dirname(__file__), 'planscore', 'tests', 'data', 'XX')

upload(prefix1, basedir1, pp.join(basedir1, '12', '*', '*.geojson'))

prefix2 = pp.join('uploads', 'sample-NC-1-992')
basedir2 = pp.join(pp.dirname(__file__), 'data', 'sample-NC-1-992')

upload(prefix2, basedir2, pp.join(basedir2, '*.*'))
upload(prefix2, basedir2, pp.join(basedir2, '*', '*.*'))

prefix3 = pp.join('uploads', 'sample-NC-1-992-simple')
basedir3 = pp.join(pp.dirname(__file__), 'data', 'sample-NC-1-992-simple')

upload(prefix3, basedir3, pp.join(basedir3, '*.*'))
upload(prefix3, basedir3, pp.join(basedir3, '*', '*.*'))

prefix4 = pp.join('uploads', 'sample-NC-1-992-incomplete')
basedir4 = pp.join(pp.dirname(__file__), 'data', 'sample-NC-1-992-incomplete')

upload(prefix4, basedir4, pp.join(basedir4, '*.*'))

prefix5 = pp.join('data', 'XX', '003')
basedir5 = pp.join(pp.dirname(__file__), 'planscore', 'tests', 'data', 'XX-sim')

upload(prefix5, basedir5, pp.join(basedir5, '12', '*', '*.geojson'))

prefix6 = pp.join('uploads', 'sample-NC5.1')
basedir6 = pp.join(pp.dirname(__file__), 'data', 'sample-NC5.1')

upload(prefix6, basedir6, pp.join(basedir6, '*.*'))
upload(prefix6, basedir6, pp.join(basedir6, '*', '*.*'))

# Lambda function setup

print('--> Set up Lambda', ENDPOINT_LAM)
lam = boto3.client('lambda', endpoint_url=ENDPOINT_LAM, region_name='us-east-1', **AWS_CREDS)

env = {
    'PLANSCORE_SECRET': 'localstack',
    'WEBSITE_BASE': 'http://127.0.0.1:5000/',
    'S3_ENDPOINT_URL': ENDPOINT_S3,
    'LAMBDA_ENDPOINT_URL': ENDPOINT_LAM,
    }

print('    Environment:', ' '.join(['='.join(kv) for kv in env.items()]))

for function_name in deploy.functions.keys():
    deploy.publish_function(lam, function_name, CODE_PATH, env, 'nobody')


"""
 Copyright (c) 2019 Intel Corporation
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
      http://www.apache.org/licenses/LICENSE-2.0
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""

from copy import deepcopy
from pathlib import Path
from typing import List, Type, Optional

import jsonschema
import jstyleson as json

from nncf.common.utils.logger import logger
from nncf.common.utils.os import safe_open
from nncf.config.schema import ROOT_NNCF_CONFIG_SCHEMA
from nncf.config.schema import validate_single_compression_algo_schema
from nncf.config.schema import validate_accuracy_aware_training_schema
from nncf.config.structures import NNCFExtraConfigStruct

class NNCFConfig(dict):
    """A regular dictionary object extended with some utility functions."""

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.__nncf_extra_structs = {}  # type: dict[str, NNCFExtraConfigStruct]

    @classmethod
    def from_dict(cls, nncf_dict):
        """
        Load NNCF config from dict;
        The dict must contain only json supported primitives.
        """

        NNCFConfig.validate(nncf_dict)
        return cls(deepcopy(nncf_dict))

    @classmethod
    def from_json(cls, path) -> 'NNCFConfig':
        file_path = Path(path).resolve()
        with safe_open(file_path) as f:
            loaded_json = json.load(f)
        return cls.from_dict(loaded_json)

    def register_extra_structs(self, struct_list: List[NNCFExtraConfigStruct]):
        for struct in struct_list:
            struct_id = struct.get_id()
            if struct_id in self.__nncf_extra_structs:
                raise RuntimeError(f'{struct_id} is already registered as extra struct in NNCFConfig!')
            self.__nncf_extra_structs[struct_id] = struct

    def get_extra_struct(self, struct_cls: Type[NNCFExtraConfigStruct]) -> NNCFExtraConfigStruct:
        return self.__nncf_extra_structs[struct_cls.get_id()]

    def has_extra_struct(self, struct_cls: Type[NNCFExtraConfigStruct]) -> NNCFExtraConfigStruct:
        return struct_cls.get_id() in self.__nncf_extra_structs

    def get_all_extra_structs_for_copy(self) -> List[NNCFExtraConfigStruct]:
        return list(self.__nncf_extra_structs.values())

    def get_redefinable_global_param_value_for_algo(self, param_name: str, algo_name: str) -> Optional:
        """
        Some parameters can be specified both on the global NNCF config .json level (so that they apply
        to all algos), and at the same time overridden in the algorithm-specific section of the .json.
        This function returns the value that should apply for a given algorithm name, considering the
        exact format of this config.

        :param param_name: The name of a parameter in the .json specification of the NNCFConfig, that may
          be present either at the top-most level of the .json, or at the top level of the algorithm-specific
          subdict.
        :param algo_name: The name of the algorithm (among the allowed algorithm names in the .json) for which
          the resolution of the redefinable parameter should occur.
        :return: The value of the parameter that should be applied for the algo specified by `algo_name`.
        """
        from nncf.config.extractors import extract_algo_specific_config
        algo_config = extract_algo_specific_config(self, algo_name)
        param = self.get(param_name)
        algo_specific_param = algo_config.get(param_name)
        if algo_specific_param is not None:
            param = algo_specific_param
        return param

    @staticmethod
    def validate(loaded_json):
        try:
            jsonschema.validate(loaded_json, schema=ROOT_NNCF_CONFIG_SCHEMA)
        except jsonschema.ValidationError as e:
            logger.error('Invalid NNCF config supplied!')

            # The default exception's __str__ result will contain the entire schema,
            # which is too large to be readable.
            import nncf.config.schema as config_schema
            msg = e.message + '. See documentation or {} for an NNCF configuration file JSON schema definition'.format(
                config_schema.__file__)
            raise jsonschema.ValidationError(msg)

        compression_section = loaded_json.get('compression')
        accuracy_aware_section = loaded_json.get('accuracy_aware_training')
        if accuracy_aware_section is not None:
            validate_accuracy_aware_training_schema(accuracy_aware_section)
        if compression_section is None:
            # No compression specified
            return

        try:
            if isinstance(compression_section, dict):
                validate_single_compression_algo_schema(compression_section)
            else:
                # Passed a list of dicts
                for compression_algo_dict in compression_section:
                    validate_single_compression_algo_schema(compression_algo_dict)
        except jsonschema.ValidationError:
            # No need to trim the exception output here since only the compression algo
            # specific sub-schema will be shown, which is much shorter than the global schema
            logger.error('Invalid NNCF config supplied!')
            raise


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @file scripts/kconfig/configure.py
#
# @author Hiroyuki Chishiro
#
"""
set up configuration.
"""

import sys
import tools

def init_sysconfig_algo(fout, algo):
  "initialize algorithm"
  fout.write("ALGO_NAME = " + algo + "\n")

def init_sysconfig_arch(fout, arch, machine):
  "initialize architecture"
  if arch == "sim":
    fout.write("ARCH_NAME = sim\n")
  elif arch == "x86_64":
    fout.write("ARCH_NAME = x86_64\n")
  elif arch == "aarch64":
    fout.write("ARCH_NAME = aarch64\n")
    if machine == "raspi3":
      fout.write("MACHINE_NAME = raspi3\n")
    elif machine == "none":
      fout.write("MACHINE_NAME = none\n")
    else:
      print("Unknown Machine: %d" % machine)
  elif arch == "axis":
    fout.write("ARCH_NAME = axis\n")
  else:
    print("Unknown Architecture: %d " % arch)

def init_sysconfig_compiler(fout, compiler):
  "initialize compiler"
  if compiler == "gcc":
    fout.write("CC = gcc\n")
  elif compiler == "clang":
    fout.write("CC = clang\n")
  else:
    print("Unknown Compiler: %d" % compiler)

def init_sysconfig_output(fout, output):
  "initialize compiler"
  if output == "console":
    fout.write("OUTPUT_NAME = console\n")
  elif output == "uart":
    fout.write("OUTPUT_NAME = uart\n")
  else:
    print("Unknown Output: %d" % output)

def init_kconfig(algo, arch, machine, compiler, output):
  "initialize kconfig"
  fout = open("./.kconfig", "w")

  init_sysconfig_algo(fout, algo)
  init_sysconfig_arch(fout, arch, machine)
  init_sysconfig_compiler(fout, compiler)
  init_sysconfig_output(fout, output)

  fout.write(tools.DEFS + "\n\n")
  fout.write(tools.CFILES + "\n\n")
  fout.write(tools.AFILES + "\n")

  fout.close()

def init_config():
  "initialize config"
  fout = open("./include/mcube/config.h", "w")
  fout.write("/**\n"
             + " * @file include/mcube/config.h\n"
             + " *\n"
             + " * @author Hiroyuki Chishiro\n"
             + " */\n")
  fout.write("#ifndef __MCUBE_MCUBE_CONFIG_H__\n"
             + "#define __MCUBE_MCUBE_CONFIG_H__\n"
             + "/* Don't edit this file.\n"
             + " * Please edit configure then do make configure. */\n")

  for key, value, in tools.CONFIGURES.items():
    if key.find("CONFIG_ARCH") == 0:
      fout.write("/** Architecture. */\n")
    elif key.find("CONFIG_COMPILER") == 0:
      fout.write("/** Compiler. */\n")
    elif key.find("CONFIG_ALGO") == 0:
      fout.write("/** Algorithm. */\n")
    elif key.find("CONFIG_PRINT") == 0:
      fout.write("/** Print. */\n")
    elif key.find("CONFIG_TQ") == 0:
      fout.write("/** Task Queue. */\n")
    elif key.find("CONFIG_TIE_BREAK") == 0:
      fout.write("/** Tie-Break. */\n")
    elif key.find("CONFIG_OPTION") == 0:
      fout.write("/** Option. */\n")
    elif key.find("CONFIG_TARGET_DRIVE") == 0:
      fout.write("/** Target Drive. */\n")
    elif key.find("CONFIG_MEMORY") == 0:
      fout.write("/** Memory. */\n")
    if value == "y":
      fout.write("#define " + key + " 1\n")
    else:
      fout.write("#define " + key + " 0\n")

  fout.write("#endif /* __MCUBE_MCUBE_CONFIG_H__ */\n")
  fout.close()

def main():
  "main function"
  configure_file = "configure"
  kconfig_file = "Kconfig"

  tools.scan_configure(configure_file)

  algo = tools.scan_algo_name(kconfig_file)
  arch = tools.scan_arch_name(kconfig_file)
  machine = tools.scan_machine_name(kconfig_file)
  compiler = tools.scan_compiler_name(kconfig_file)
  output = tools.scan_output_name(kconfig_file)

  tools.scan_kconfig(kconfig_file)

  if not tools.check_exclusives(kconfig_file):
    sys.exit("Error: check_exclusives()")

  tools.scan_dependencies(kconfig_file)

  if not tools.check_conflicts_and_dependencies():
    sys.exit("Error: check_conflicts_and_dependencies()")

  init_kconfig(algo, arch, machine, compiler, output)
  init_config()

if __name__ == "__main__":
  main()


import numpy as np
import matplotlib.pylab as plt

def step(x):
    y = x>0
    return y.astype(np.int)

def sigmoid(x):
    return 1/(1+np.exp(-x))

def relu(x):
    return np.maximum(0,x)

def identity_function(x):
    return x

x = np.arange(-5.0 , 5.0 , 0.01)
y = step(x)

plt.plot(x,y)
plt.ylim(-0.1,1.1)
plt.grid(1)
plt.show()

X = np.array([1.0, 0.5])
W1 = np.array([[0.1, 0.3, 0.5],[0.2, 0.4, 0.6]])
B1 = np.array([0.1, 0.2, 0.3])

A1 = np.dot(X, W1) + B1
Z1 = sigmoid(A1)

W2 = np.array([[0.1,0.4],[0.2,0.5],[0.3,0.6]])
B2 = np.array([0.1,0.2])

A2 = np.dot(Z1, W2) + B2
Z2 = sigmoid(A2)

W3 = np.array([[0.1, 0.3],[0.2,0.4]])
B3 = np.array([0.1,0.2])

A3 = np.dot(Z2,W3) + B3
Y = identity_function(A3)