from flask import Flask, jsonify
import sqlalchemy
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import create_engine, func
from datetime import datetime, timedelta
from itertools import chain
from flask import render_template, redirect, url_for, request, send_from_directory, flash

# create engine to hawaii.sqlite
engine = create_engine("sqlite:///Resources/gun_dataset.sqlite")
# reflect an existing database into a new model
Base = automap_base()
# reflect the tables
Base.prepare(engine, reflect=True)

# View all of the classes that automap found
Base.classes.keys()

# Save references to each table
gun_sale = Base.classes.Gun_sale
gun_violence_2019 = Base.classes.Gun_violence_2019
gun_violence_2020 = Base.classes.Gun_violence_2020
gun_violence_2021 = Base.classes.Gun_violence_2021
completed_data=Base.classes.completed_final_data
state_coords=Base.classes.state_coords

#Set up Flask
app = Flask(__name__)

@app.route("/")

def welcome():
    html=f"""This is the main page.

 These links are for demo only.  If you use this as an api, route directly to the link that returns your data

  In this case the "get all of the data" link

  If you are running this locally remember that you will want to remove /dev from the url.

 Gun Sale data by State

 gun violence mass shooting on 2019

 gun violence mass shooting on 2020

 gun violence mass shooting on 2021

  Completed data includ accurate Lat & Lon from 2019 to 2021

  State Coords
            """
    return html

# Convert the query results to a Dictionary using date as the key and prcp as the value.
@app.route("/api/v1.0/gunsalebystate")
def gunsalebystate():
    list_gunsale=[]
    session = Session(engine)
    Gun_sale_data= session.query(gun_sale.State, gun_sale.Year_2017,gun_sale.Year_2018,gun_sale.Year_2019,
                                gun_sale.Year_2020,gun_sale.Year_2021).all()
    session.close()

    for data in Gun_sale_data:
        dict_sale={}
        dict_sale['State'] = data[0]
        dict_sale['feature'] = {}

        dic_state={}
        dic_state['2017'] = data[1]
        dic_state['2018'] = data[2]
        dic_state['2019'] = data[3]
        dic_state['2020'] = data[4]
        dic_state['2021'] = data[5]

        dict_sale['feature'] = dic_state
        list_gunsale.append(dict_sale)

    return jsonify(list_gunsale)

@app.route("/api/v1.0/gunviolence2019")
def gunviolence_2019():
    list_gv19=[]
    session = Session(engine)
    Gun_violence_2019_data= session.query(gun_violence_2019.IncidentID, gun_violence_2019.State, gun_violence_2019.CityOrCounty,
                                        gun_violence_2019.Address,gun_violence_2019.Killed, gun_violence_2019.Injured, gun_violence_2019.Date,
                                        gun_violence_2019.Year, gun_violence_2019.Latitude, gun_violence_2019.Longitude).all()
    session.close()

    for data in Gun_violence_2019_data:
        dic_gv19={}
        dic_gv19['IncidentID']=data[0]
        dic_gv19['feature']={}

        dic_gvf19={}
        dic_gvf19['State']=data[1]
        dic_gvf19['City']=data[2]
        dic_gvf19['Address']=data[3]
        dic_gvf19['Killed']=data[4]
        dic_gvf19['Injured']=data[5]
        dic_gvf19['Date']=data[6]
        dic_gvf19['Year']=data[7]
        dic_gvf19['Latitude']=data[8]
        dic_gvf19['Longitude']=data[9]
        dic_gv19['feature']=dic_gvf19
        list_gv19.append(dic_gv19)

    return jsonify(list_gv19)

@app.route("/api/v1.0/gunviolence2020")
def gunviolence_2020():
    list_gv20=[]
    session = Session(engine)
    Gun_violence_2020_data= session.query(gun_violence_2020.IncidentID, gun_violence_2020.State, gun_violence_2020.CityOrCounty,
                                        gun_violence_2020.Address,gun_violence_2020.Killed, gun_violence_2020.Injured, gun_violence_2020.Date,
                                        gun_violence_2020.Year, gun_violence_2020.Latitude, gun_violence_2020.Longitude).all()
    session.close()

    for data in Gun_violence_2020_data:
        dic_gv20={}
        dic_gv20['IncidentID']=data[0]
        dic_gv20['feature']={}

        dic_gvf20={}
        dic_gvf20['State']=data[1]
        dic_gvf20['City']=data[2]
        dic_gvf20['Address']=data[3]
        dic_gvf20['Killed']=data[4]
        dic_gvf20['Injured']=data[5]
        dic_gvf20['Date']=data[6]
        dic_gvf20['Year']=data[7]
        dic_gvf20['Latitude']=data[8]
        dic_gvf20['Longitude']=data[9]
        dic_gv20['feature']=dic_gvf20
        list_gv20.append(dic_gv20)

    return jsonify(list_gv20)

@app.route("/api/v1.0/gunviolence2021")
def gunviolence_2021():
    list_gv21=[]
    session = Session(engine)
    Gun_violence_2021_data= session.query(gun_violence_2021.IncidentID, gun_violence_2021.State, gun_violence_2021.CityOrCounty,
                                        gun_violence_2021.Address,gun_violence_2021.Killed, gun_violence_2021.Injured, gun_violence_2021.Date,
                                        gun_violence_2021.Year, gun_violence_2021.Latitude, gun_violence_2021.Longitude).all()
    session.close()

    for data in Gun_violence_2021_data:
        dic_gv21={}
        dic_gv21['IncidentID']=data[0]
        dic_gv21['feature']={}

        dic_gvf21={}
        dic_gvf21['State']=data[1]
        dic_gvf21['City']=data[2]
        dic_gvf21['Address']=data[3]
        dic_gvf21['Killed']=data[4]
        dic_gvf21['Injured']=data[5]
        dic_gvf21['Date']=data[6]
        dic_gvf21['Year']=data[7]
        dic_gvf21['Latitude']=data[8]
        dic_gvf21['Longitude']=data[9]
        dic_gv21['feature']=dic_gvf21
        list_gv21.append(dic_gv21)

    return jsonify(list_gv21)

@app.route("/api/v1.0/completedata")
def complete_data():
    list_cfd=[]
    session = Session(engine)
    completed_dataset= session.query(completed_data.IncidentID, completed_data.State, completed_data.CityOrCounty,
                                        completed_data.StreetAddress,completed_data.Full_Address ,completed_data.Killed, completed_data.Injured,
                                        completed_data.Date, completed_data.Year, completed_data.Latitude, completed_data.Longitude).all()
    session.close()

    for data in completed_dataset:
        dic_cfd={}
        dic_cfd['IncidentID']=data[0]
        dic_cfd['feature']={}

        dic_cfdf={}
        dic_cfdf['State']=data[1]
        dic_cfdf['City']=data[2]
        dic_cfdf['Address']=data[3]
        dic_cfdf['Full Address']=data[4]
        dic_cfdf['Killed']=data[5]
        dic_cfdf['Injured']=data[6]
        dic_cfdf['Date']=data[7]
        dic_cfdf['Year']=data[8]
        dic_cfdf['Latitude']=data[9]
        dic_cfdf['Longitude']=data[10]
        dic_cfd['feature']=dic_cfdf
        list_cfd.append(dic_cfd)

    return jsonify(list_cfd)

@app.route("/api/v1.0/statecoords")
def statecoord_data():
    list_sc=[]
    session = Session(engine)
    state_coords_dataset= session.query(state_coords.State, state_coords.Latitude, state_coords.Longitude).all()
    session.close()

    for data in state_coords_dataset:
        dic_sc={}
        dic_sc['State']=data[0]
        dic_sc['feature']={}

        dic_scf={}
        dic_scf['Latitude']=data[1]
        dic_scf['Longitude']=data[2]
        dic_sc['feature']=dic_scf
        list_sc.append(dic_sc)

    return jsonify(list_sc)

if __name__ == '__main__':
    app.run(debug=True)


"""Example how to plot sky regions on a sky image.
"""
from astropy.coordinates import SkyCoord, Angle
from regions import (
    make_example_dataset,
    CircleSkyRegion,
)
import matplotlib.pyplot as plt

config = dict(crpix=(18, 9), cdelt=(-10, 10), shape=(18, 36))
dataset = make_example_dataset(data='simulated', config=config)
wcs = dataset.wcs

fig = plt.figure()
ax = fig.add_axes([0.15, 0.1, 0.8, 0.8], projection=wcs)

ax.set_xlim(-0.5, dataset.config['shape'][1] - 0.5)
ax.set_ylim(-0.5, dataset.config['shape'][0] - 0.5)

ax.imshow(dataset.image.data, cmap='gray', vmin=0, vmax=1,
          interpolation='nearest', origin='lower')

for source in dataset.source_table:
    # Plot a sky circle around each source
    center = SkyCoord(source['GLON'], source['GLAT'], unit='deg', frame='galactic')
    radius = Angle(20, 'deg')
    region = CircleSkyRegion(center=center, radius=radius)
    pix_region = region.to_pixel(wcs=wcs)

    pix_region.plot(ax=ax, edgecolor='yellow', facecolor='yellow', alpha=0.5, lw=3)


#!/usr/bin/python
#
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Utilities for our job runner, for working with configs.
"""

from __future__ import absolute_import, division, print_function

import argparse
import os
import sys
from enum import Enum
from typing import Any, Dict, List, Optional

import commentjson
import yaml

import caliban.platform.cloud.types as ct

class JobMode(str, Enum):
  CPU = 'CPU'
  GPU = 'GPU'

# Special config for Caliban.
CalibanConfig = Dict[str, Any]

DRY_RUN_FLAG = "--dry_run"
CALIBAN_CONFIG = ".calibanconfig.json"

# Defaults for various input values that we can supply given some partial set
# of info from the CLI.
DEFAULT_REGION = ct.US.central1

# : Dict[JobMode, ct.MachineType]
DEFAULT_MACHINE_TYPE = {
    JobMode.CPU: ct.MachineType.highcpu_32,
    JobMode.GPU: ct.MachineType.standard_8
}
DEFAULT_GPU = ct.GPU.P100

# Config to supply for CPU jobs.
DEFAULT_ACCELERATOR_CONFIG = {
    "count": 0,
    "type": "ACCELERATOR_TYPE_UNSPECIFIED"
}

def gpu(job_mode: JobMode) -> bool:
  """Returns True if the supplied JobMode is JobMode.GPU, False otherwise.

  """
  return job_mode == JobMode.GPU

def load_yaml_config(path):
  """returns the config parsed based on the info in the flags.

  Grabs the config file, written in yaml, slurps it in.
  """
  with open(path) as f:
    config = yaml.load(f, Loader=yaml.FullLoader)

  return config

def load_config(path, mode='yaml'):
  """Load a JSON or YAML config.

  """
  if mode == 'json':
    with open(path) as f:
      return commentjson.load(f)

  return load_yaml_config(path)

def valid_json(path: str) -> Dict[str, Any]:
  """Loads JSON if the path points to a valid JSON file; otherwise, throws an
  exception that's picked up by argparse.

  """
  try:
    return load_config(path, mode='json')
  except commentjson.JSONLibraryException:
    raise argparse.ArgumentTypeError(
        """File '{}' doesn't seem to contain valid JSON. Try again!""".format(
            path))

def extract_script_args(m: Dict[str, Any]) -> List[str]:
  """Strip off the "--" argument if it was passed in as a separator."""
  script_args = m.get("script_args")
  if script_args is None or script_args == []:
    return script_args

  head, *tail = script_args

  return tail if head == "--" else script_args

def extract_project_id(m: Dict[str, Any]) -> str:
  """Attempts to extract the project_id from the args; falls back to an
  environment variable, or exits if this isn't available. There's no sensible
  default available.

  """
  project_id = m.get("project_id") or os.environ.get("PROJECT_ID")

  if project_id is None:
    print()
    print(
        "\nNo project_id found. 'caliban cloud' requires that you either set a \n\
$PROJECT_ID environment variable with the ID of your Cloud project, or pass one \n\
explicitly via --project_id. Try again, please!")
    print()

    sys.exit(1)

  return project_id

def extract_region(m: Dict[str, Any]) -> ct.Region:
  """Returns the region specified in the args; defaults to an environment
  variable. If that's not supplied defaults to the default cloud provider from
  caliban.platform.cloud.

  """
  region = m.get("region") or os.environ.get("REGION")

  if region:
    return ct.parse_region(region)

  return DEFAULT_REGION

def extract_zone(m: Dict[str, Any]) -> str:
  return "{}-a".format(extract_region(m))

def extract_cloud_key(m: Dict[str, Any]) -> Optional[str]:
  """Returns the Google service account key filepath specified in the args;
  defaults to the $GOOGLE_APPLICATION_CREDENTIALS variable.

  """
  return m.get("cloud_key") or \
    os.environ.get("GOOGLE_APPLICATION_CREDENTIALS")

def apt_packages(conf: CalibanConfig, mode: JobMode) -> List[str]:
  """Returns the list of aptitude packages that should be installed to satisfy
  the requests in the config.

  """
  packages = conf.get("apt_packages") or {}

  if isinstance(packages, dict):
    k = "gpu" if gpu(mode) else "cpu"
    return packages.get(k, [])

  elif isinstance(packages, list):
    return packages

  else:
    raise argparse.ArgumentTypeError(
        """{}'s "apt_packages" entry must be a dictionary or list, not '{}'""".
        format(CALIBAN_CONFIG, packages))

def caliban_config() -> CalibanConfig:
  """Returns a dict that represents a `.calibanconfig.json` file if present,
  empty dictionary otherwise.
  """
  if not os.path.isfile(CALIBAN_CONFIG):
    return {}

  with open(CALIBAN_CONFIG) as f:
    conf = commentjson.load(f)
    return conf


from ..preprocessing import filter_and_normalize, moments
from . import velocity, velocity_graph, velocity_embedding

def run_all(data, basis=None, mode='deterministic', min_counts=3, n_pcs=30, n_neighbors=30, copy=False):
    from time import time
    start = time()

    adata = data.copy() if copy else data
    filter_and_normalize(adata, min_counts=min_counts)
    print("Number of genes to be used:", adata.X.shape[1])
    moments(adata, n_neighbors=n_neighbors, n_pcs=n_pcs)

    velocity(adata, mode=mode)
    print("Number of genes to be used:", adata.var.velocity_genes.sum())

    velocity_graph(adata)
    if basis is not None: velocity_embedding(adata, basis=basis)

    print('Total time (seconds): ' + str(round(time() - start, 2)))

    return adata if copy else None

def convert_to_loom(adata, basis=None):
    from scipy.sparse import issparse
    import numpy as np
    import velocyto

    class VelocytoLoom(velocyto.VelocytoLoom):
        def __init__(self, adata, basis=None):
            self.S = adata.layers['spliced'].T
            self.U = adata.layers['unspliced'].T
            self.S = self.S.A if issparse(self.S) else self.S
            self.U = self.U.A if issparse(self.U) else self.U

            if 'ambiguous' in adata.layers.keys():
                self.A = adata.layers['ambiguous'].T
                if issparse(self.A): self.A = self.A.A

            self.initial_cell_size = self.S.sum(0)
            self.initial_Ucell_size = self.U.sum(0)

            self.ca = dict()
            self.ra = dict()
            for key in adata.obs.keys():
                self.ca[key] = np.array(adata.obs[key].values)
            for key in adata.var.keys():
                self.ra[key] = np.array(adata.var[key].values)
            if 'X_' + basis in adata.obsm.keys():
                self.ts = adata.obsm['X_' + basis]

            if 'clusters' in self.ca:
                self.set_clusters(self.ca['clusters'])
            elif 'louvain' in self.ca:
                self.set_clusters(self.ca['louvain'])

        def filter_and_normalize(self, min_counts=3, min_counts_u=3, n_top_genes=None):
            # counterpart to scv.pp.filter_and_normalize()
            self.score_detection_levels(min_expr_counts=min_counts, min_expr_counts_U=min_counts_u,
                                        min_cells_express=0, min_cells_express_U=0)
            self.filter_genes(by_detection_levels=True)

            self.normalize('both', size=True, log=False)
            if n_top_genes is not None:
                self.score_cv_vs_mean(n_top_genes)
                self.filter_genes(by_cv_vs_mean=True)

            self.normalize_by_total()
            print("Number of genes to be used:", self.S.shape[0])

        def impute(self, n_pcs=30, n_neighbors=30):
            # counterpart to scv.pp.moments()
            self.perform_PCA(n_components=n_pcs)
            self.knn_imputation(k=n_neighbors, n_pca_dims=n_pcs)
            self.normalize_median()

        def velocity(self, fit_offset=False):
            self.fit_gammas(fit_offset=fit_offset)
            self.filter_genes_good_fit()

            self.predict_U()
            self.calculate_velocity()
            self.calculate_shift()
            self.extrapolate_cell_at_t()
            print("Number of genes to be used:", self.S.shape[0])

        def velocity_graph(self, n_neighbors_graph=100):
            if not hasattr(self, 'ts'):
                raise ValueError('Compute embedding first.')
            else:
                # counterpart to scv.tl.velocity_graph()
                self.estimate_transition_prob(hidim="Sx_sz", embed="ts", transform="linear",
                                              n_neighbors=n_neighbors_graph, knn_random=True, sampled_fraction=1)

                # counterpart to scv.tl.velocity_embedding()
                self.calculate_embedding_shift(sigma_corr=0.1, expression_scaling=True)

        def run_all(self, min_counts=3, min_counts_u=3, n_pcs=30, n_neighbors=30, n_neighbors_graph=100,
                    n_top_genes=None, fit_offset=False):
            from time import time
            start = time()

            self.filter_and_normalize(min_counts=min_counts, min_counts_u=min_counts_u, n_top_genes=n_top_genes)
            print('Preprocessing: ' + str(round(time() - start, 2)))
            timestamp = time()

            self.impute(n_pcs=n_pcs, n_neighbors=n_neighbors)
            print('Imputation: ' + str(round(time() - timestamp, 2)))
            timestamp = time()

            self.velocity(fit_offset=fit_offset)
            print('Velocity Estimation: ' + str(round(time() - timestamp, 2)))
            timestamp = time()

            self.velocity_graph(n_neighbors_graph=n_neighbors_graph)
            print('Velocity Graph: ' + str(round(time() - timestamp, 2)))

            print('Total: ' + str(round(time() - start, 2)))

    return VelocytoLoom(adata, basis)


from __future__ import absolute_import, unicode_literals
import logging
from celery import shared_task
from stats import views as stats_views

logger = logging.getLogger(__name__)

warm_views = [
    stats_views.CompanyTypeCountView,
    stats_views.TopKvedsView,
    stats_views.RegisteredCompaniesCountView,
    stats_views.RegisteredFopsCountView,
    stats_views.PepsCountView,
    stats_views.PepRelatedPersonsCountView,
    stats_views.PepLinkedCompaniesCountView,
    stats_views.PepRelationCategoriesCountView,
]

def cache_warm_up():
    logger.info('Start cache warming')
    for view in warm_views:
        view.warm_up_cache()
        logger.info(f'{view.get_cache_key()} warmed up successfully')
    logger.info('End cache warming')

@shared_task
def cache_warming_up():
    cache_warm_up()


"""
Support embedding version number lookup into cli
"""
# Import python libs
import importlib
import sys

CONFIG = {
    "version": {
        "default": False,
        "action": "store_true",
        "help": "Display version information",
    }
}

def run(hub, primary):
    """
    Check the version number and then exit
    """
    mod = importlib.import_module(f"{primary}.version")
    print(f"{primary} {mod.version}")
    sys.exit(0)


from flask import Flask
from celery import Celery
import os

app = Flask(__name__)
simple_app = Celery('simple_worker',
                    backend = os.environ.get('CELERY_RESULT_BACKEND'),
                    broker = os.environ.get('CELERY_BROKER_URL'))

@app.route('/simple_start_task')
def call_method():
    app.logger.info("Invoking Method ")
    #                        queue name in task folder.function name
    r = simple_app.send_task('tasks.longtime_add', kwargs={'x': 1, 'y': 2})
    app.logger.info(r.backend)
    return r.id

@app.route('/simple_task_status/<task_id>')
def get_status(task_id):
    status = simple_app.AsyncResult(task_id, app=simple_app)
    print("Invoking Method ")
    return "Status of the Task " + str(status.state)

@app.route('/simple_task_result/<task_id>')
def task_result(task_id):
    result = simple_app.AsyncResult(task_id).result
    return "Result of the Task " + str(result)


"""
Bar chart demo with pairs of bars grouped for easy comparison.
"""
import sys
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
import matplotlib.patches as mpatches

def multibar():
    n_groups = 10
    data = [[ 152, 281, 127, 43,  120,210,74,13,1,0], ## big
            [ 712, 543, 587, 543, 493,481,478,406,573,952], ## equal
            ]
    small = (136, 176, 286, 414, 387,309,448,581,426,48)

    fig, ax = plt.subplots()

    opacity = 0.4
    error_config = {'ecolor': '0.3'}

    ## single bar

    columns = ('$Class_{1}$', '$Class_{2}$', '$Class_{3}$', '$Class_{4}$', '$Class_{5}$','$Class_{6}$','$Class_{7}$','$Class_{8}$','$Class_{9}$','$Class_{10}$')

    # Get some pastel shades for the colors

    index = np.arange(len(columns))
    bar_width = 0.4

    # Initialize the vertical-offset for the stacked bar chart.
    y_offset = np.zeros(len(columns))
    #for row in range(2):## single bar have 2 part.
    plt.bar(index, data[0], bar_width, bottom=y_offset, color='#99FFFF', label = '$max_i > CP_i$')
    y_offset = y_offset + data[0]
    plt.bar(index, data[1], bar_width, bottom=y_offset, color='#00CC00',label = '$max_i = CP_i$')

    rects2 = ax.bar(index + bar_width, small, bar_width,
                    alpha=opacity, color='r',
                    error_kw=error_config,
                    label='$max_i$ < $CP_{i}$')
    ## set label name
    ax.set_xlabel('Class')
    ax.set_ylabel('Number of comparisons')
    ax.set_title('Comparison of CP set and Max set ')

    ## set ticks size
    ax.set_xticks(index + bar_width / 2)
    ax.set_xticklabels(columns)

    ## set class_i size
    #ax.xaxis.set_tick_params(labelsize=28)

    ## set legend position
    ax.legend(loc = 'best')

    ## set font
    '''
    font = {'family' : 'normal',
                'weight' : 'bold',
                'size'   : 20}
    plt.rc('font',**font)
    '''

    fig.tight_layout()

    ## save svg file
    #plt.savefig("/home/newslab/Pictures/schedule/test.svg")
    plt.show()

if __name__=="__main__":
    fo = open('resultComparison.txt','r')

    multibar()


import datetime as dt
import os
from resources import moon

from flask import Flask
app = Flask(__name__)

@app.route('/phase')
@app.route('/phase/<int:year>/<int:month>/<int:day>')
def phase(year=None, month=None, day=None):
    if year is None and month is None and day is None:
        today = dt.datetime.now()
        today -= dt.timedelta(days=1) # 2020 leap day correction
        year, month, day = today.year, today.month, today.day
    p = (moon.julian(year, month, day) - moon.julian(2000, 1, 6)) % 29.530588853

    if p < 1.84566:
        return "🌑"  # new
    elif p < 5.53699:
        return "🌒"  # waxing crescent
    elif p < 9.22831:
        return "🌓"  # first quarter
    elif p < 12.91963:
        return "🌔"  # waxing gibbous
    elif p < 16.61096:
        return "🌕"  # full
    elif p < 20.30228:
        return "🌖"  # waning gibbous
    elif p < 23.99361:
        return "🌗"  # last quarter
    elif p < 27.68493:
        return "🌘"  # waning crescent
    else:
        return "🌑"  # new


from django.shortcuts import render, get_object_or_404
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.views import View
from .forms import SubmitUrlForm
from .models import ShortenDB
from django.urls import reverse

# Create your views here.

class Home(View):
    def get(self, request, *args, **kwargs):
        form = SubmitUrlForm()
        title = "Hey! There, This is vit.ly"
        context = {
            "title": title,
            "form": form
        }
        return render(request, 'URLShortener/home.html', context)

    def post(self, request, *args, **kwargs):
        form = SubmitUrlForm(request.POST)
        title = "Submit a Valid URL"
        context = {
            "title": title,
            "form": form
        }
        template = 'URLShortener/home.html'
        if form.is_valid():
            new_url = form.cleaned_data.get("url")
            if 'http' not in new_url:
                new_url = 'http://'+new_url
            obj, created = ShortenDB.objects.get_or_create(url=new_url)
            context = {
                "object": obj,
                "created": created,
            }
            if created:
                template = "URLShortener/success.html"
            else:
                template = "URLShortener/already-exists.html"

        return render(request, template, context)

class URLRedirectView(View):
    def get(self, request, shortcode=None, *args, **kwargs):
        obj = get_object_or_404(ShortenDB, shortcode__iexact=shortcode)
        return HttpResponseRedirect(obj.url)


from kfp.components import InputPath, OutputPath, create_component_from_func

def catboost_train_classifier(
    training_data_path: InputPath('CSV'),
    model_path: OutputPath('CatBoostModel'),
    starting_model_path: InputPath('CatBoostModel') = None,
    label_column: int = 0,

    loss_function: str = 'Logloss',
    num_iterations: int = 500,
    learning_rate: float = None,
    depth: int = 6,
    random_seed: int = 0,

    cat_features: list = None,
    text_features: list = None,

    additional_training_options: dict = {},
):
    '''Train a CatBoost classifier model.

    Args:
        training_data_path: Path for the training data in CSV format.
        model_path: Output path for the trained model in binary CatBoostModel format.
        starting_model_path: Path for the existing trained model to start from.
        label_column: Column containing the label data.

        loss_function: The metric to use in training and also selector of the machine learning
            problem to solve. Default = 'Logloss'
        num_iterations: Number of trees to add to the ensemble.
        learning_rate: Step size shrinkage used in update to prevents overfitting.
            Default value is selected automatically for binary classification with other parameters set to default.
            In all other cases default is 0.03.
        depth: Depth of a tree. All trees are the same depth. Default = 6
        random_seed: Random number seed. Default = 0

        cat_features: A list of Categorical features (indices or names).
        text_features: A list of Text features (indices or names).
        additional_training_options: A dictionary with additional options to pass to CatBoostClassifier

    Outputs:
        model: Trained model in binary CatBoostModel format.

    Annotations:
        author: Alexey Volkov <alexey.volkov@ark-kun.com>
    '''
    import tempfile
    from pathlib import Path

    from catboost import CatBoostClassifier, Pool

    column_descriptions = {label_column: 'Label'}
    column_description_path = tempfile.NamedTemporaryFile(delete=False).name
    with open(column_description_path, 'w') as column_description_file:
        for idx, kind in column_descriptions.items():
            column_description_file.write('{}\t{}\n'.format(idx, kind))

    train_data = Pool(
        training_data_path,
        column_description=column_description_path,
        has_header=True,
        delimiter=',',
    )

    model = CatBoostClassifier(
        iterations=num_iterations,
        depth=depth,
        learning_rate=learning_rate,
        loss_function=loss_function,
        random_seed=random_seed,
        verbose=True,
        **additional_training_options,
    )

    model.fit(
        train_data,
        cat_features=cat_features,
        text_features=text_features,
        init_model=starting_model_path,
        #verbose=False,
        #plot=True,
    )
    Path(model_path).parent.mkdir(parents=True, exist_ok=True)
    model.save_model(model_path)

if __name__ == '__main__':
    catboost_train_classifier_op = create_component_from_func(
        catboost_train_classifier,
        output_component_file='component.yaml',
        base_image='python:3.7',
        packages_to_install=['catboost==0.23'],
        annotations={
            "author": "Alexey Volkov <alexey.volkov@ark-kun.com>",
            "canonical_location": "https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/CatBoost/Train_classifier/from_CSV/component.yaml",
        },
    )


import numpy as np
import cv2

img=cv2.imread('faces.jpg')

gray=cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)

template = cv2.imread('face_template2.jpg',cv2.IMREAD_GRAYSCALE)

w,h=template.shape[::-1]
method=eval('cv2.TM_CCOEFF')

res=cv2.matchTemplate(gray,template,method)

min_val,max_val,min_loc,max_loc=cv2.minMaxLoc(res)

top_left=max_loc

bottom_right=(top_left[0]+w,top_left[1]+h)

cv2.rectangle(img,top_left,bottom_right,255,2)

cv2.imshow('cv2.TM_CCOEFF',img)

cv2.waitKey(0)

cv2.destoryAllWindows()


import discord
from cassiopeia import riotapi
from cassiopeia.type.api.exception import APIError
from discord.ext import commands
from sqlite3 import OperationalError
import config
import database
import utilities

class Summoner:
    """Commands relating to individual summoners."""

    def __init__(self, bot):
        self.bot = bot
        riotapi.set_api_key(config.API)

    async def raise_exception(self, ctx, exception: str, sum_name: str, region: str):
        """HTTP error handling"""
        if exception.error_code == 400:
            embed = discord.Embed(
                title="400: Bad Request!",
                description="Please join the support server with b!support.",
                colour=0xCA0147)
            utilities.footer(ctx, embed)
            await ctx.send("", embed=embed)
        elif exception.error_code == 403:
            embed = discord.Embed(
                title="403: Forbidden!",
                description="Most likely my API key has expired",
                colour=0xCA0147)
            utilities.footer(ctx, embed)
            await ctx.send("", embed=embed)
        elif exception.error_code == 404:
            embed = discord.Embed(
                title="404: Not Found!",
                description="Could not find summoner '{0}' on {1}".format(sum_name, region),
                colour=0xCA0147)
            utilities.footer(ctx, embed)
            await ctx.send("", embed=embed)
        elif exception.error_code == 415:
            embed = discord.Embed(
                title="415: Unsupported Media Type!",
                description="I have no clue how you triggered this one.",
                colour=0xCA0147)
            utilities.footer(ctx, embed)
            await ctx.send("", embed=embed)
        elif exception.error_code == 429:
            embed = discord.Embed(
                title="429: Rate Limit Exceeded!",
                description="Please try again later",
                colour=0xCA0147)
            utilities.footer(ctx, embed)
            await ctx.send("", embed=embed)
        elif exception.error_code == 500:
            embed = discord.Embed(
                title="500: Internal Server Error!",
                description="Please try again later.",
                colour=0xCA0147)
            utilities.footer(ctx, embed)
            await ctx.send("", embed=embed)
        elif exception.error_code == 503:
            embed = discord.Embed(
                title="503: Service Unavailable!",
                description="Please try again later.",
                colour=0xCA0147)
            utilities.footer(ctx, embed)
            await ctx.send("", embed=embed)

    @commands.command(ignore_extra=False, no_pm=True)
    async def search(self, ctx, sum_name: str, region=None):
        """'Summoner Name' '[optional] Region'"""
        if region is None:
            try:
                db = database.Database('guilds.db')
                region = db.find_entry(ctx.guild.id)
                db.close_connection()
            except TypeError:
                embed = utilities.error_embed(ctx, "Please specify a region, or set a default region with `b!region set [region]`.")
                await ctx.send("", embed=embed)
                return

        if "'" in sum_name:
            embed = utilities.error_embed(ctx, "Please use quotation marks to enclose names")
            await ctx.send("", embed=embed)
            return

        await ctx.trigger_typing()

        try:
            riotapi.set_region(region)
        except ValueError:
            embed = utilities.error_embed(ctx, "{0} is not a valid region! Valid regions are listed in `b!region list`.".format(region))
            await ctx.send("", embed=embed)
            return

        try:
            summoner = riotapi.get_summoner_by_name(sum_name)
            leagues = riotapi.get_league_entries_by_summoner(summoner)
            top_champ = riotapi.get_top_champion_masteries(summoner, max_entries=3)
        except APIError as exception:
            await Summoner.raise_exception(self, ctx, exception, sum_name, region)
            return

        db = database.Database("guilds.db")
        user = db.find_user(str(ctx.guild.id), sum_name)
        if user is None:
            db.add_user(str(ctx.guild.id), sum_name, region)
            db.close_connection()

        embed = discord.Embed(colour=0x1affa7)
        top_champs = "{0}, {1} and {2}".format(top_champ[0].champion.name, top_champ[1].champion.name, top_champ[2].champion.name)
        icon_url = 'http://ddragon.leagueoflegends.com/cdn/6.24.1/img/profileicon/{}.png'.format(summoner.profile_icon_id)
        overall_wins, overall_losses = 0, 0

        for league in leagues:
            queue = league.queue.value
            tier = league.tier.value
            for entries in league.entries:
                division = entries.division.value
                league_points = str(entries.league_points) + ' LP'
                wins = entries.wins
                losses = entries.losses
                overall_wins += wins
                overall_losses += losses
                ratio = (wins / (wins + losses) * 100)

            if queue == 'RANKED_SOLO_5x5':
                embed.add_field(name="Ranked Solo:", value=u'\u200B', inline=True)
                embed.add_field(name="Division",
                                value="{0} {1} - {2}".format(tier, division, league_points),
                                inline=True)
                embed.add_field(name="W/L",
                                value="{0}W - {1}L ({2:.0F}%)".format(wins, losses, ratio),
                                inline=True)
                embed.add_field(name=u"\u200B", value=u"\u200B", inline=False)
            elif queue == 'RANKED_FLEX_SR':
                embed.add_field(name="Ranked Flex:", value=u'\u200B', inline=True)
                embed.add_field(name="Division",
                                value="{0} {1} - {2}".format(tier, division, league_points),
                                inline=True)
                embed.add_field(name="W/L",
                                value="{0}W - {1}L ({2:.0F}%)".format(wins, losses, ratio),
                                inline=True)
                embed.add_field(name=u"\u200B", value=u"\u200B", inline=False)
            elif queue == 'RANKED_FLEX_TT':
                embed.add_field(name="Ranked TT:", value=u'\u200B', inline=True)
                embed.add_field(name="Division",
                                value="{0} {1} - {2}".format(tier, division, league_points),
                                inline=True)
                embed.add_field(name="W/L",
                                value="{0}W - {1}L ({2:.0F}%)".format(wins, losses, ratio),
                                inline=True)
                embed.add_field(name=u"\u200B", value=u"\u200B", inline=False)

            overall_ratio = (overall_wins / (overall_wins + overall_losses) * 100)

        overall = "{0}W/{1}L ({2:.2f})%".format(overall_wins, overall_losses, overall_ratio)
        op_gg = "https://{0}.op.gg/summoner/userName={1}".format(region, sum_name.replace(" ", "%20"))
        embed.set_author(name="Summoner Lookup - {0} ({1})".format(sum_name, region), url=op_gg, icon_url=icon_url)
        embed.add_field(name="Overall:", value=u'\u200B', inline=True)
        embed.add_field(name="Top Champions", value=top_champs, inline=True)
        embed.add_field(name="W/L", value=overall, inline=True)
        utilities.footer(ctx, embed)
        await ctx.send("", embed=embed)

    @commands.command(ignore_extra=False, no_pm=True)
    async def mastery(self, ctx, sum_name: str, champ_name: str, region=None):
        """'Summoner Name' 'Champion Name' '[optional] Region'"""
        if region is None:
            try:
                db = database.Database('guilds.db')
                region = db.find_entry(ctx.guild.id)
                db.close_connection()
            except TypeError:
                embed = utilities.error_embed(ctx, "Please specify a region, or set a default region with `b!region set [region]`.")
                await ctx.send("", embed=embed)
                return

        if "'" in sum_name or "'" + champ_name + "'" in champ_name:
            embed = utilities.error_embed(ctx, "Please use double quotes to enclose names.")
            await ctx.send("", embed=embed)
            return

        try:
            riotapi.set_region(region)
        except ValueError:
            embed = utilities.error_embed(ctx, "{0} is not a valid region! Valid regions are listed in `b!region list`.".format(region))
            await ctx.send("", embed=embed)
            return

        try:
            summoner = riotapi.get_summoner_by_name(sum_name)
            champion = riotapi.get_champion_by_name(champ_name)
            mastery = riotapi.get_champion_mastery(summoner, champion)
        except APIError as exception:
            await Summoner.raise_exception(self, ctx, exception, sum_name, region)
            return
        except AttributeError:
            embed = utilities.error_embed(ctx, "Could not find champion '{0}'. Please remember capitals.".format(champ_name))
            utilities.footer(ctx, embed)
            await ctx.send("", embed=embed)
            return

        embed = discord.Embed(colour=0x1AFFA7)
        op_gg = "https://{0}.op.gg/summoner/userName={1}".format(region, sum_name.replace(" ", "%20"))
        icon_url = utilities.fix_url(champ_name)
        embed.set_author(name="{0} Mastery - {1} ({2})".format(champion.name, summoner.name, region), url=op_gg, icon_url=icon_url)
        embed.add_field(name="Champion Level:", value=mastery.level, inline=True)
        embed.add_field(name="Mastery Points:", value=mastery.points, inline=True)
        embed.add_field(name="Points to next level:", value=mastery.points_until_next_level, inline=True)
        utilities.footer(ctx, embed)
        await ctx.send("", embed=embed)

    @commands.command(no_pm=True)
    @commands.is_owner()
    async def game(self, ctx, sum_name: str, region: str):

        if region is None:
            try:
                db = database.Database('guilds.db')
                region = db.find_entry(ctx.guild.id)
                db.close_connection()
            except TypeError:
                embed = utilities.error_embed(ctx, "Please specify a region, or set a default region with `b!region set [region]`.")
                await ctx.send("", embed=embed)
                return

        if "'" in sum_name:
            embed = utilities.error_embed(ctx, "Please use quotation marks to enclose names")
            await ctx.send("", embed=embed)
            return

        await ctx.trigger_typing()

        try:
            riotapi.set_region(region)
        except ValueError:
            embed = utilities.error_embed(ctx, "{0} is not a valid region! Valid regions are listed in `b!region list`.".format(region))
            await ctx.send("", embed=embed)
            return

        try:
            summoner = riotapi.get_summoner_by_name(sum_name)
            game = riotapi.get_current_game(summoner)
        except APIError as exception:
            await Summoner.raise_exception(self, ctx, exception, sum_name, region)
            return

        try:
            does_game_exist = game.id
        except AttributeError:
            embed = utilities.error_embed(ctx, "{} is not in an active game!".format(summoner.name))
            await ctx.send("", embed=embed)
            return

def setup(bot):
    bot.add_cog(Summoner(bot))

    # More commands from API


import torch
import random
import os
import torchvision.transforms as transforms
from PIL import Image

class CelebaDataset(object):
    def __init__(self,opt):
        self.opt = opt
        self.celeba_imgpaths = self.get_img_path()
        self.size = len(self.celeba_imgpaths)
        self.transform = transforms.Compose(
            [transforms.CenterCrop(opt.full_size),
            transforms.ToTensor(),
            transforms.Normalize((0.5, 0.5, 0.5),
                                (0.5, 0.5, 0.5))]
        )

    def get_img_path(self):
        imgpaths = []
        count = 0
        assert os.path.isdir(self.opt.datadir),'%s不是目录'%self.opt.datadir
        for root,_,fnames in os.walk(self.opt.datadir):
            for fname in fnames:
                path = os.path.join(root,fname)
                imgpaths.append(path)
                if self.opt.max_dataset != 0:
                    count += 1
                    if count == self.opt.max_dataset:
                        break
        return imgpaths
    def __getitem__(self,index):
        path = self.celeba_imgpaths[index % self.size]
        img = Image.open(path).convert('RGB')
        img = self.transform(img)
        return img
    def __len__(self):
        return self.size

class UnalignedDataset(object):
    def __init__(self, opt):
        self.opt = opt
        self.root = opt.dataroot
        self.dir_A = os.path.join(opt.dataroot, opt.phase + 'A')
        self.dir_B = os.path.join(opt.dataroot, opt.phase + 'B')

        self.A_paths = self.get_img_path(self.dir_A)
        self.B_paths = self.get_img_path(self.dir_B)

        self.A_size = len(self.A_paths)
        self.B_size = len(self.B_paths)

        self.coord = self.get_coord()
        self.transform = transforms.Compose(
            [transforms.RandomCrop(opt.fineSize),
            transforms.RandomHorizontalFlip(),
            transforms.ToTensor(),
            transforms.Normalize((0.5, 0.5, 0.5),
                                (0.5, 0.5, 0.5))]
        )
    def get_img_path(self,dir):
        imgpaths = []
        count = 0
        assert os.path.isdir(dir),'%s不是目录'%dir
        for root,_,fnames in os.walk(dir):
            for fname in fnames:
                path = os.path.join(root,fname)
                imgpaths.append(path)
                if self.opt.max_dataset != 0:
                    count += 1
                    if count == self.opt.max_dataset:
                        break
        return imgpaths

    def __getitem__(self, index):
        A_path = self.A_paths[index % self.A_size]
        B_path = self.B_paths[index % self.B_size]
        A_img = Image.open(A_path).convert('RGB')
        B_img = Image.open(B_path).convert('RGB')
        A_img = self.transform(A_img)
        B_img = self.transform(B_img)
        if random.random() < 0.5:
            times = random.randint(200,400)/100.
            input_img = (A_img+1)/2./times
            input_img = input_img*2-1
        else:
            input_img = A_img

        r,g,b = input_img[0]+1, input_img[1]+1, input_img[2]+1
        A_gray = 1. - (0.299*r+0.587*g+0.114*b)/2.
        A_gray = torch.unsqueeze(A_gray, 0)

        # A_img = torch.cat((A_img,self.coord),1)
        # B_img = torch.cat((B_img,self.coord),1)
        # A_gray = torch.cat((A_gray,self.coord),1)
        # input_img = torch.cat((input_img,self.coord),1)

        # A_img = self.crop(A_img)
        # B_img = self.crop(B_img)
        # A_gray = self.crop(A_gray)
        # input_img = self.crop(input_img)
        return {'A': A_img, 'B': B_img, 'A_gray': A_gray, 'input_img': input_img}

    def __len__(self):
        return max(self.A_size, self.B_size)

    def get_coord(self):
        size = self.opt.fineSize
        x = torch.linspace(-1,1,size)
        x = x.view(-1,1)
        x = x.expand(-1,size)
        y = torch.linspace(-1,1,size)
        y = y.view(1,-1)
        y = y.expand(size,-1)
        coord = torch.stack((x,y),0)
        return coord

    def crop(self,x,index,size):
        wh = int(self.opt.fineSize/size)
        i = index//wh
        j = index%wh
        return x[:,:,i*size:i*size+size,j*size:j*size+size]


import requests
import argparse
import json

def get_credentials(fieldname):
    return json.loads(open(get_config_name(), "r").read())[fieldname]

def get_config_name():
    parser = argparse.ArgumentParser()
    parser.add_argument("config_name", nargs="?")
    config_name = parser.parse_args().config_name
    if not config_name:
        config_name = "config.json"
    return config_name

def get_html(url):
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
        "Accept-Language": "en-US, en;q=0.5",
    }
    return requests.get(url, headers=headers).text

def send_telegram(message):
    request_string = (
        "https://api.telegram.org/bot"
        + get_credentials("telegram")["bot_token"]
        + "/sendMessage?chat_id="
        + get_credentials("telegram")["user_id"]
        + "&parse_mode=Markdown&text="
        + message
    )
    response = requests.get(request_string)
    return response.json()

sites = get_credentials("watched_sites")
for site in sites:
    html = get_html(site["url"])
    if site["search_snippet"] not in html:
        message = f"👁  state {site['title']} changed: {site['url']}"
        send_telegram(message)


from flask import Blueprint, request, render_template, \
                  flash, g, session, redirect, url_for

from flask.ext.login import current_user, logout_user, login_user
from app import db, lm, app
from app.auth.forms import LoginForm, SignupForm
from app.main.forms import SearchForm
from app.main.models import User

# Define the blueprint: 'auth', set its url prefix: app.url/auth
mod_auth = Blueprint('auth', __name__, url_prefix='/auth')

@lm.user_loader
def load_user(id):
    return User.query.get(int(id))

@app.before_request
def before_request():
    g.user = current_user
    if g.user.is_authenticated:
        g.search_form = SearchForm()

# Set the route and accepted methods
@mod_auth.route('/login/', methods=['GET', 'POST'])
def login():

    if g.user is not None and g.user.is_authenticated:
        return redirect(url_for('main.profile'))
    form = LoginForm()
    if form.validate_on_submit():
        user = User.query.filter_by(email=form.email.data).first()
        login_user(user, form.remember_me.data)
        return redirect(url_for('main.profile'))

    return render_template('auth/login.html', form=form)

@mod_auth.route('/signup/', methods=['GET', 'POST'])
def signup():
    """
    signup view
    """
    form = SignupForm(request.form)

    if request.method == 'POST':
        if form.validate() == False:
            return render_template('auth/signup.html', form=form)
        else:
            newuser = User(form.firstname.data,
                           form.lastname.data,
                           form.username.data,
                           form.email.data,
                           form.password.data)
            db.session.add(newuser)
            db.session.commit()

            session['email'] = newuser.email
            return redirect(url_for('main.profile'))

    elif request.method == 'GET':
        return render_template('auth/signup.html', form=form)

@mod_auth.route('/logout/')
def logout():
    logout_user()
    flash('You have successfully logged out.')
    return redirect(url_for('main.index'))


import time
from werobot.client import Client
from werobot.robot import WeRoBot
from werobot.utils import cached_property

from redis_util import redis_client
from config import logger

class RewriteAccessTokenClient(Client):

    def get_access_token(self):
        token_ttl = redis_client.get_access_token_ttl()
        if token_ttl < 60:
            identifier = redis_client.acquire_lock_with_timeout("wp_access_token", acquire_timeout=10, lock_timeout=10)
            try:
                if identifier:
                    if redis_client.get_access_token_ttl() < 60:
                        json = self.grant_token()
                        access_token = json["access_token"]
                        expire = json["expires_in"]
                        redis_client.set_access_token(access_token, expire)
            finally:
                redis_client.release_access_token_lock(identifier)

        return redis_client.get_access_token()

class Robot(WeRoBot):

    @cached_property
    def client(self):
        logger.info("use RewriteAccessTokenClient")
        return RewriteAccessTokenClient(self.config)


from math import ceil
from os import path, mkdir

import xlsxwriter

def xlsx_writer(file_name, file_data):
    """
    Cria e escreve um arquivo xlsx contendo tabelas com as informaçẽos passadas
    :param file_name: nome do arquivo a ser gerado
    :param file_data: dicionário contando as informações a serem escritas no
    arquivo
    :return: None
    """

    # path do diretório onde arquivo será salvo
    dir_name = path.join(path.dirname(__file__), 'Resultados')
    # path do arquivo
    file_name = path.join(dir_name, f'{file_name}.xlsx')

    if not path.exists(dir_name):  # verifica a existência do diretório
        mkdir(dir_name)  # cria o diretório

    with xlsxwriter.Workbook(file_name) as workbook:  # cria o arquivo
        for sheet_name, sheet_data in file_data.items():
            worksheet = workbook.add_worksheet(name=sheet_name)  # cria página
            worksheet.set_column('A:B', 25)  # formata largura das colunas A e B

            for i, row in enumerate(sheet_data.items()):
                # define tamanho da altura da linha como proporcional ao
                # tamanho do texto contido nela
                size = ceil(len(''.join(row[1].split())) / 25)
                size = ceil(size / 1.5)
                size = 30 * size

                # define cor da célula
                color = '#b1b1b1' if i % 2 else '#747474'

                # define formatação da ceĺula
                cell_format = workbook.add_format({
                    'text_wrap': True,
                    'align': 'center',
                    'valign': 'vcenter',
                    'border': 6,
                    'bg_color': color
                })

                worksheet.set_row(i, size)  # formata altura da linha

                # escreve conteúdo da primeira coluna da linha
                worksheet.write(i, 0, row[0], cell_format)

                # dict contendo relação entre scores e cor da célula
                scores = {
                    'RA1000': '#4ad851',
                    'ÓTIMO': '#5fbe64',
                    'BOM': '#3c6b9c',
                    'REGULAR': '#d2d84a',
                    'RUIM': '#fb1b00',
                    'NÃO RECOMENDADA': '#8735bd'
                }

                # verifica se conteúdo da segunda coluna é um score
                if row[1] in scores:
                    #  reformata a célula de acordo com o score
                    cell_format = workbook.add_format({
                        'text_wrap': True,
                        'align': 'center',
                        'valign': 'vcenter',
                        'border': 6,
                        'bold': True,
                        'bg_color': scores[row[1]]
                    })

                # escreve conteúdo da segunda coluna da linha
                worksheet.write(i, 1, row[1], cell_format)


import tkinter as tk
from  tkinter import messagebox
import numpy as np
from PIL import Image, ImageTk
import os
import time
import json
import random
from colors import COLOR_TABLE

WIDTH = 3

class FrameCanvas(tk.Frame):
    def __init__(self, master, height, width, app):
        super().__init__(master)

        self.master = master
        self.canvas_h = height
        self.canvas_w = width
        self.app = app
        self.image_path = None
        self.label_path = None

        # 绘布
        self.canvas = None

        # 图像
        self.image = None # 原始图像
        self.image_view = None # 显示图像, 定义防止被回收
        self.image_view_tk = None
        self.image_canvas = None

        # 图形控制
        self.scale = 1.0
        self.scale_step = 0.05

        # 标签
        # [   [  ymin, xmin, ymax, xmax, lb, [ [y, x, y, x, lb], .. ], [ lb, lb, lb, .. ]  ],   ...   ]
        self.labels = []

        # 状态
        self.rect_canvas = None # 当前框
        self.selected_color = None
        self.selected = False
        self.selected_sub = False
        self.ymin = 0
        self.xmin = 0
        self.ymax = 0
        self.xmax = 0

        # 指示点
        self.p1_canvas = None
        self.p2_canvas = None
        self.p3_canvas = None
        self.p4_canvas = None
        self.idx = None # 当前选中框
        self.idx_selected = None

        self.create_canvas()

    def create_canvas(self):
        self.canvas = tk.Canvas(self.master, bg='black', height=self.canvas_h, width=self.canvas_w)
        self.canvas.bind("<MouseWheel>", self.callback_mouse_wheel) # 滚动调整图像大小
        self.canvas.bind("<Button-1>", self.callback_mouse_btn1) # 两次左键加入一个矩形
        self.canvas.bind("<Motion>", self.callback_mouse_move) # 两次左键之间的拖动绘制矩形
        self.canvas.bind("<Button-3>", self.callback_mouse_btn2) # 右键取消选中
        self.master.bind('<Escape>', self.callback_escape) # 删除选定框
        self.master.bind('<space>', self.callback_space) # 添加二级框
        self.canvas.focus_set()
        self.canvas.place(x=0, y=0, anchor='nw')

    def callback_space(self, event):

        if self.idx is not None:
            self.selected_sub = True
            self.idx_selected = self.idx

    def callback_mouse_btn1(self, event):
        if self.image is None:
            return
        if not self.selected:
            self.selected = True
            self.selected_color = COLOR_TABLE[random.randint(0, len(COLOR_TABLE)-1)]
            self.ymin, self.xmin = event.y, event.x
        else:
            self.selected = False
            if not self.selected_sub:
                lb = self.app.entry_anns.get().split('-')[0].strip()
                ymin, ymax = min(self.ymin, self.ymax), max(self.ymin, self.ymax)
                xmin, xmax = min(self.xmin, self.xmax), max(self.xmin, self.xmax)
                self.labels.append([
                    round(ymin/self.scale),
                    round(xmin/self.scale),
                    round(ymax/self.scale),
                    round(xmax/self.scale), lb, [], []])
            else:
                # TODO
                lb = self.app.entry_anns.get().split('-')[1].split('|')[0].strip()
                ymin, ymax = min(self.ymin, self.ymax), max(self.ymin, self.ymax)
                xmin, xmax = min(self.xmin, self.xmax), max(self.xmin, self.xmax)
                self.labels[self.idx_selected][5].append([
                    round(ymin/self.scale),
                    round(xmin/self.scale),
                    round(ymax/self.scale),
                    round(xmax/self.scale), lb])
                self.idx_selected = None
                self.selected_sub = False
            self.canvas.delete(self.rect_canvas)
            self.rect_canvas = None
            self._set_labels_to_background(self.labels[-1], self.selected_color)
            self.app._list_labels(self.labels)

    def callback_mouse_btn2(self, event):
        self.selected = False
        self.canvas.delete(self.rect_canvas)
        self.rect_canvas = None

    def callback_mouse_move(self, event):
        if self.image is not None:
            idx = self._idx_inside_box(event.y, event.x)
            if idx != self.idx:
                self._clear_4points()
                self._draw_4points(idx)
                self.idx = idx
            if self.selected:
                self.canvas.delete(self.rect_canvas)
                self.rect_canvas = None
                self.ymax, self.xmax = event.y, event.x
                self.rect_canvas = self.canvas.create_rectangle(
                    self.xmin, self.ymin, self.xmax, self.ymax, outline=self.selected_color, width=WIDTH)

    def save(self):
        if (self.image_path is not None) and (self.label_path is not None):
            with open(self.label_path, 'w') as load_f:
                json.dump(self.labels, load_f)
                print('Save: ', self.label_path)

    def load(self, image_path, label_path):
        self.image = Image.open(image_path)
        w, h = self.image.size
        self.scale = float(self.canvas_h) / h
        ow = round(self.scale*w)
        if ow > self.canvas_w:
            self.scale = float(self.canvas_w) / w
        find = os.path.exists(label_path)
        if find:
            with open(label_path, 'r') as load_f:
                self.labels = list(json.load(load_f))
        else:
            self.labels = []
            with open(label_path, 'w') as load_f:
                json.dump(self.labels, load_f)
        # TODO:
        self.image_path = image_path
        self.label_path = label_path
        self._refresh_canvas()
        return self.labels

    def callback_mouse_wheel(self, event):
        if self.image is None:
            return
        if event.delta > 0:
            self.scale += self.scale_step
        else:
            self.scale -= self.scale_step
            if self.scale < 0:
                self.scale = 0
        self._refresh_canvas()

    def callback_escape(self, event):
        print('Delete: ', self.idx)
        self._delete_box()
        self._refresh_canvas()
        self.app._list_labels(self.labels)

    def _refresh_canvas(self):
        '''
        Param:    self.image, self.scale, self.canvas
        Temp:     self.image_view, self.image_view_tk, self.image_canvas
        '''
        self.canvas.delete(tk.ALL)
        self.rect_canvas = None
        self.selected = False
        self._set_background()
        self._set_labels_to_background()

    def _set_background(self):
        '''
        Param:    self.image, self.scale, self.canvas
        Temp:     self.image_view, self.image_view_tk
        Recreate: self.image_canvas
        '''
        self.image_view = self.image.copy()
        w, h = self.image_view.size
        w, h = int(w*self.scale), int(h*self.scale)
        self.image_view = self.image_view.resize((w, h))
        self.image_view_tk = ImageTk.PhotoImage(self.image_view)
        self.canvas.delete(self.image_canvas)
        self.image_canvas = self.canvas.create_image(0, 0, image=self.image_view_tk, anchor='nw')

    def _set_labels_to_background(self, elem=None, color=None):
        '''
        Param:    self.labels, self.scale, self.canvas
        Recreate: self.canvas->rect

        [   [  ymin, xmin, ymax, xmax, lb, [ [y, x, y, x, lb], .. ], [ lb, lb, lb, .. ]  ],   ...   ]
        '''
        s = self.scale
        if elem is None:
            labels = self.labels
            fixed = False
            self.canvas.delete('Rectangle')
        else:
            labels = [elem]
            fixed = True
        if self.image is not None:
            for label in labels:
                if not fixed:
                    color = COLOR_TABLE[random.randint(0, len(COLOR_TABLE)-1)]
                _ymin, _xmin, _ymax, _xmax = int(label[0]*s), int(label[1]*s), int(label[2]*s), int(label[3]*s)
                _lb = str(label[4])
                self.canvas.create_rectangle(_xmin, _ymin, _xmax, _ymax, outline=color, width=WIDTH)
                for label_sub in label[5]:
                    ymin, xmin, ymax, xmax = \
                        int(label_sub[0]*s), int(label_sub[1]*s), int(label_sub[2]*s), int(label_sub[3]*s)
                    lb = str(label_sub[4])
                    self.canvas.create_rectangle(xmin, ymin, xmax, ymax, outline=color, width=WIDTH)
                    self.canvas.create_text(xmin+5, ymin+8, text=lb, fill=color, anchor='w')
                for label_sub in label[6]:
                    lb = str(label_sub)
                    _lb += '|' + lb
                self.canvas.create_text(_xmin+5, _ymin+8, text=_lb, fill=color, anchor='w')

    def _idx_inside_box(self, y, x):
        '''
        Param:  self.labels
        Return: index or None
        '''
        s = self.scale
        for i in range(len(self.labels)):
            label = self.labels[i]
            ymin, xmin, ymax, xmax = \
                int(label[0]*s), int(label[1]*s), int(label[2]*s), int(label[3]*s)
            dy1, dx1 = y - ymin, x - xmin
            dy2, dx2 = ymax - y, xmax - x
            if (dy1>0) and (dy2>0) and (dx1>0) and (dx2>0):
                return i
        return None

    def _draw_4points(self, idx):
        '''
        Param: self.labels, box idx
        Create: self.p1_canvas, self.p2_canvas, self.p3_canvas, self.p4_canvas, self.canvas
        '''
        if idx is None:
            return
        if self.p1_canvas is None:
            s = self.scale
            label = self.labels[idx]
            wd = 5
            ymin, xmin, ymax, xmax = \
                int(label[0]*s), int(label[1]*s), int(label[2]*s), int(label[3]*s)
            self.p1_canvas = \
                self.canvas.create_rectangle(xmin - wd, ymin - wd, xmin + wd, ymin + wd, fill='red')
            self.p2_canvas = \
                self.canvas.create_rectangle(xmin - wd, ymax - wd, xmin + wd, ymax + wd, fill='red')
            self.p3_canvas = \
                self.canvas.create_rectangle(xmax - wd, ymin - wd, xmax + wd, ymin + wd, fill='red')
            self.p4_canvas = \
                self.canvas.create_rectangle(xmax - wd, ymax - wd, xmax + wd, ymax + wd, fill='red')

    def _clear_4points(self):
        if self.p1_canvas is not None:
            self.canvas.delete(self.p1_canvas)
            self.canvas.delete(self.p2_canvas)
            self.canvas.delete(self.p3_canvas)
            self.canvas.delete(self.p4_canvas)
            self.p1_canvas = None
            self.p2_canvas = None
            self.p3_canvas = None
            self.p4_canvas = None

    def _delete_box(self):
        '''
        Param:  self.idx
        Modify: self.labels, self.idx
        '''
        if self.idx is not None:
            label = self.labels[self.idx]
            self.labels.remove(label)
            self.idx = None


# Author: Bichen Wu (bichen@berkeley.edu) 08/25/2016

"""Train"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import cv2
from datetime import datetime
import os.path
import sys
import time
import argparse
import tempfile
import json
import shutil

import numpy as np
from six.moves import xrange
import tensorflow as tf

from config import *
from dataset import *
from utils.util import sparse_to_dense, bgr_to_rgb, bbox_transform
from nets import *
import nx_commons.model_evaulation.iou_engine as iou_engine

JSON_PREFIX = '.json'

def _draw_box(im, box_list, label_list, color=(0,255,0), cdict=None, form='center'):
  assert form == 'center' or form == 'diagonal', \
      'bounding box format not accepted: {}.'.format(form)

  for bbox, label in zip(box_list, label_list):

    if form == 'center':
      bbox = bbox_transform(bbox)

    xmin, ymin, xmax, ymax = [int(b) for b in bbox]

    l = label.split(':')[0] # text before "CLASS: (PROB)"
    if cdict and l in cdict:
      c = cdict[l]
    else:
      c = color

    # draw box
    cv2.rectangle(im, (xmin, ymin), (xmax, ymax), c, 1)
    # draw label
    font = cv2.FONT_HERSHEY_SIMPLEX
    cv2.putText(im, label, (xmin, ymax), font, 0.3, c, 1)

def _viz_prediction_result(model, images, bboxes, labels, batch_det_bbox,
                           batch_det_class, batch_det_prob):
  mc = model.mc

  for i in range(len(images)):
    # draw ground truth
    _draw_box(
        images[i], bboxes[i],
        [mc.dataset.CLASS_NAMES[idx] for idx in labels[i]],
        (0, 255, 0))

    # draw prediction
    det_bbox, det_prob, det_class = model.filter_prediction(
        batch_det_bbox[i], batch_det_prob[i], batch_det_class[i])

    keep_idx    = [idx for idx in range(len(det_prob)) \
                      if det_prob[idx] > mc.post_processing.PLOT_PROB_THRESH]
    det_bbox    = [det_bbox[idx] for idx in keep_idx]
    det_prob    = [det_prob[idx] for idx in keep_idx]
    det_class   = [det_class[idx] for idx in keep_idx]

    _draw_box(
        images[i], det_bbox,
        [mc.dataset.CLASS_NAMES[idx]+': (%.2f)'% prob \
            for idx, prob in zip(det_class, det_prob)],
        (0, 0, 255))

def train(mcfg):
  """Train SqueezeDet model"""
  available_models = { 'squeezeDet' : SqueezeDet,
                       'squeezeDet+' : SqueezeDetPlus,
                       'resnet50' : ResNet50ConvDet }

  available_datasets = { 'NEXAREAR' : nexarear }
  assert mcfg.dataset.DATASET in available_datasets.keys(), \
      'Selected dataset not supported: {}'.format(mcfg.dataset.DATASET)

  with tf.Graph().as_default():

    assert mcfg.base_net in available_models.keys() , \
        'Selected neural net architecture not supported: {}'.format(mcfg.base_net)

    model = available_models[mcfg.base_net](mcfg)
    imdb = available_datasets[mcfg.dataset.DATASET](model.ANCHOR_BOX,mcfg)

    BATCH_SIZE = mcfg.train.BATCH_SIZE
    checkpoint_dir = mcfg.train.CHECKPOINT_DIR
    train_logs = mcfg.train.LOG_TRAIN_DIR

    N_ANCHORS = len(model.ANCHOR_BOX)
    N_CLASSES = len(mcfg.dataset.CLASS_NAMES)
    KEEP_PROB = mcfg.train.KEEP_PROB
    DEBUG_MODE = mcfg.DEBUG_MODE
    MAX_STEPS = mcfg.train.MAX_STEPS
    CHECKPOINT_STEP = mcfg.train.CHECKPOINT_STEP
    SUMMARY_STEP = mcfg.train.SUMMARY_STEP
    NUM_OF_CHECKPOINTS_TO_KEEP = mcfg.train.NUM_OF_CHECKPOINTS_TO_KEEP

    NUM_OF_TEST_ITERATIONS = mcfg.test.NUM_OF_TEST_ITERATIONS
    test_logs = mcfg.test.LOG_TEST_DIR
    IOU_THRESOLD= mcfg.test.IOU_THRESHOLD

    # save model size, flops, activations by layers
    with open(os.path.join(checkpoint_dir, 'model_metrics.txt'), 'w') as f:
      f.write('Number of parameter by layer:\n')
      count = 0
      for c in model.model_size_counter:
        f.write('\t{}: {}\n'.format(c[0], c[1]))
        count += c[1]
      f.write('\ttotal: {}\n'.format(count))

      count = 0
      f.write('\nActivation size by layer:\n')
      for c in model.activation_counter:
        f.write('\t{}: {}\n'.format(c[0], c[1]))
        count += c[1]
      f.write('\ttotal: {}\n'.format(count))

      count = 0
      f.write('\nNumber of flops by layer:\n')
      for c in model.flop_counter:
        f.write('\t{}: {}\n'.format(c[0], c[1]))
        count += c[1]
      f.write('\ttotal: {}\n'.format(count))
    f.close()
    print ('Model statistics saved to {}.'.format(
      os.path.join(checkpoint_dir, 'model_metrics.txt')))

    saver = tf.train.Saver(tf.global_variables(), max_to_keep=NUM_OF_CHECKPOINTS_TO_KEEP)
    summary_op = tf.summary.merge_all()
    init = tf.global_variables_initializer()

    ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
    if ckpt and ckpt.model_checkpoint_path:
        saver.restore(sess, ckpt.model_checkpoint_path)

    sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
    sess.run(init)
    tf.train.start_queue_runners(sess=sess)

    summary_writer = tf.summary.FileWriter(train_logs, sess.graph)
    test_summary_writer = tf.summary.FileWriter(test_logs, sess.graph)

    with tf.variable_scope('Test_Model') as scope:
        precision = tf.placeholder(tf.float32, name='precision')
        precision_op = tf.summary.scalar('precision_summary', precision)

        num_of_detections = tf.placeholder(tf.int32, name='num_of_detections')
        num_of_detections_op = tf.summary.scalar('num_of_detections_summary', num_of_detections)

        localization_error_precentage = tf.placeholder(tf.float32, name='localization_error_precentage')
        localization_error_precentage_op = tf.summary.scalar('localization_error_precentage_summary', localization_error_precentage)

        classification_error_precentage = tf.placeholder(tf.float32, name='classification_error_precentage')
        classification_error_precentage_op = tf.summary.scalar('classification_error_precentage_summary', classification_error_precentage)

        background_error_precentage = tf.placeholder(tf.float32, name='background_error_precentage')
        background_error_precentage_op = tf.summary.scalar('background_error_precentage_summary', background_error_precentage)

        repeated_error_precentage = tf.placeholder(tf.float32, name='repeated_error_precentage')
        repeated_error_precentage_op = tf.summary.scalar('repeated_error_precentage_summary', repeated_error_precentage)

        recall = tf.placeholder(tf.float32, name='recall')
        recall_op = tf.summary.scalar('recall_summary', recall)

    for step in xrange(MAX_STEPS):
      start_time = time.time()

      # read batch input
      image_per_batch, label_per_batch, box_delta_per_batch, aidx_per_batch, \
          bbox_per_batch = imdb.read_batch()

      label_indices, bbox_indices, box_delta_values, mask_indices, box_values, \
          = [], [], [], [], []
      aidx_set = set()
      num_discarded_labels = 0
      num_labels = 0
      for i in range(len(label_per_batch)): # batch_size
        for j in range(len(label_per_batch[i])): # number of annotations
          num_labels += 1
          if (i, aidx_per_batch[i][j]) not in aidx_set:
            aidx_set.add((i, aidx_per_batch[i][j]))
            label_indices.append(
                [i, aidx_per_batch[i][j], label_per_batch[i][j]])
            mask_indices.append([i, aidx_per_batch[i][j]])
            bbox_indices.extend(
                [[i, aidx_per_batch[i][j], k] for k in range(4)])
            box_delta_values.extend(box_delta_per_batch[i][j])
            box_values.extend(bbox_per_batch[i][j])
          else:
            num_discarded_labels += 1

      if DEBUG_MODE:
        print ('Warning: Discarded {}/({}) labels that are assigned to the same'
               'anchor'.format(num_discarded_labels, num_labels))

      feed_dict = {
          model.image_input: image_per_batch,
          model.keep_prob: KEEP_PROB,
          model.input_mask: np.reshape(
              sparse_to_dense(
                  mask_indices, [BATCH_SIZE, N_ANCHORS],
                  [1.0]*len(mask_indices)),
              [BATCH_SIZE, N_ANCHORS, 1]),
          model.box_delta_input: sparse_to_dense(
              bbox_indices, [BATCH_SIZE, N_ANCHORS, 4],
              box_delta_values),
          model.box_input: sparse_to_dense(
              bbox_indices, [BATCH_SIZE, N_ANCHORS, 4],
              box_values),
          model.labels: sparse_to_dense(
              label_indices,
              [BATCH_SIZE, N_ANCHORS, N_CLASSES],
              [1.0]*len(label_indices)),
      }

      if step % SUMMARY_STEP == 0:
        pred_json_folder = tempfile.mkdtemp()
        # test model
        for ii_test in range(NUM_OF_TEST_ITERATIONS):
          test_img_batch, scales_batch, img_fnames_batch = imdb.read_test_image_batch()
          ground_truth_boxes_directory = imdb.get_label_path()
          test_feed_dict = {
              model.image_input: test_img_batch,
              model.keep_prob: KEEP_PROB,
              model.input_mask: np.reshape(
                  sparse_to_dense(
                      mask_indices, [BATCH_SIZE, N_ANCHORS],
                      [1.0] * len(mask_indices)),
                  [BATCH_SIZE, N_ANCHORS, 1]),
              model.box_delta_input: sparse_to_dense(
                  bbox_indices, [BATCH_SIZE, N_ANCHORS, 4],
                  box_delta_values),
              model.box_input: sparse_to_dense(
                  bbox_indices, [BATCH_SIZE, N_ANCHORS, 4],
                  box_values),
              model.labels: sparse_to_dense(
                  label_indices,
                  [BATCH_SIZE, N_ANCHORS, N_CLASSES],
                  [1.0] * len(label_indices)),
          }
          compute_time = infer_bounding_boxes_on_image_batch(model, sess, test_feed_dict, img_fnames_batch,
                                                             pred_json_folder)
          print('Processing time batch {} {}'.format(ii_test, compute_time))
          # extract score
        results = iou_engine.get_bbox_average_iou_evaulation(ground_truth_boxes_directory, pred_json_folder,
                                                           imdb.classes, IOU_THRESOLD, in_images_dir=None,
                                                           out_images_and_boxes_dir=None)
        print('Model Eval Score {}'.format(results))
        clean_folders([pred_json_folder])

        model_eval_summary_feed_dict = {num_of_detections: results['num_of_detections'],
                                      precision: results['precision'],
                                      localization_error_precentage: results['localization_error_precentage'],
                                      classification_error_precentage: results['classification_error_precentage'],
                                      background_error_precentage: results['background_error_precentage'],
                                      repeated_error_precentage: results['repeated_error_precentage'],
                                      recall: results['recall']}
        model_validation_summary = sess.run(
          [num_of_detections_op, precision_op, localization_error_precentage_op, classification_error_precentage_op,
           background_error_precentage_op, repeated_error_precentage_op, recall_op],
          feed_dict=model_eval_summary_feed_dict)

        for val_stats in model_validation_summary:
          test_summary_writer.add_summary(val_stats, step)

        op_list = [
            model.train_op, model.loss, summary_op, model.det_boxes,
            model.det_probs, model.det_class, model.conf_loss,
            model.bbox_loss, model.class_loss
        ]
        _, loss_value, summary_str, det_boxes, det_probs, det_class, conf_loss, \
            bbox_loss, class_loss = sess.run(op_list, feed_dict=feed_dict)

        _viz_prediction_result(
            model, image_per_batch, bbox_per_batch, label_per_batch, det_boxes,
            det_class, det_probs)
        image_per_batch = bgr_to_rgb(image_per_batch)
        viz_summary = sess.run(
            model.viz_op, feed_dict={model.image_to_show: image_per_batch})

        num_discarded_labels_op = tf.summary.scalar(
            'counter/num_discarded_labels', num_discarded_labels)
        num_labels_op = tf.summary.scalar(
            'counter/num_labels', num_labels)

        counter_summary_str = sess.run([num_discarded_labels_op, num_labels_op])

        summary_writer.add_summary(summary_str, step)
        summary_writer.add_summary(viz_summary, step)
        for sum_str in counter_summary_str:
          summary_writer.add_summary(sum_str, step)

        print ('conf_loss: {}, bbox_loss: {}, class_loss: {}'.
            format(conf_loss, bbox_loss, class_loss))
      else:
        _, loss_value, conf_loss, bbox_loss, class_loss = sess.run(
            [model.train_op, model.loss, model.conf_loss, model.bbox_loss,
             model.class_loss], feed_dict=feed_dict)

      duration = time.time() - start_time

      assert not np.isnan(loss_value), \
          'Model diverged. Total loss: {}, conf_loss: {}, bbox_loss: {}, ' \
          'class_loss: {}'.format(loss_value, conf_loss, bbox_loss, class_loss)

      if step % 10 == 0:
        num_images_per_step = BATCH_SIZE
        images_per_sec = num_images_per_step / duration
        sec_per_batch = float(duration)
        format_str = ('%s: step %d, loss = %.2f (%.1f images/sec; %.3f '
                      'sec/batch)')
        print (format_str % (datetime.now(), step, loss_value,
                             images_per_sec, sec_per_batch))
        sys.stdout.flush()

      # Save the model checkpoint periodically.
      if step % CHECKPOINT_STEP == 0 or (step + 1) == MAX_STEPS:
        checkpoint_path = os.path.join(checkpoint_dir, 'model.ckpt')
        saver.save(sess, checkpoint_path, global_step=step)

def clean_folders(folders_lst):
    for folder in folders_lst:
        if os.path.isdir(folder):
            shutil.rmtree(folder)

def infer_bounding_boxes_on_image_batch(model, sess, test_feed_dict, img_fnames_batch, pred_json_folder):

    t_start = time.time()

    IMAGE_HEIGHT = model.mc.dataset.IMAGE_HEIGHT
    IMAGE_WIDTH = model.mc.dataset.IMAGE_WIDTH

    det_boxes, det_probs, det_class = sess.run(
        [model.det_boxes, model.det_probs, model.det_class],
        feed_dict=test_feed_dict)
    test_img_batch = test_feed_dict[model.image_input]
    for ii_img, img in enumerate(test_img_batch):
        width = img.shape[1]
        height = img.shape[0]

        DW = float(float(width) / float(IMAGE_WIDTH))
        DH = float(float(height) / float(IMAGE_HEIGHT))

        final_boxes, final_probs, final_class = model.filter_prediction(
            det_boxes[ii_img], det_probs[ii_img], det_class[ii_img])

        compute_time = time.time() - t_start

        keep_idx = [idx for idx in range(len(final_probs)) \
                    if final_probs[idx] > model.mc.post_processing.PLOT_PROB_THRESH]
        final_boxes = [final_boxes[idx] for idx in keep_idx]
        final_probs = [final_probs[idx] for idx in keep_idx]
        final_class = [final_class[idx] for idx in keep_idx]

        box_list = final_boxes
        label_list = [model.mc.dataset.CLASS_NAMES[idx] + ': (%.2f)' % prob \
             for idx, prob in zip(final_class, final_probs)]

        all_out_boxes = []
        for bbox, label in zip(box_list, label_list):
            cx, cy, w, h = bbox
            xmin_sdet = cx - w / 2
            ymin_sdet = cy - h / 2
            xmax_sdet = cx + w / 2
            ymax_sdet = cy + h / 2

            xmin = float(xmin_sdet * DW)
            ymin = float(ymin_sdet * DH)
            xmax = float(xmax_sdet * DW)
            ymax = float(ymax_sdet * DH)

            class_label = label.split(':')[0]  # text before "CLASS: (PROB)"

            out_box = { "type": "RECT",
                        "label": class_label,
                        "position": "UNDEFINED",
                        "bounding_box_with_pose": {  "p0": { 'x' : xmin, 'y' : ymin},
                                                     "p1": { 'x' : xmax, 'y' : ymax},
                                                     "width": float(w*DW),
                                                     "height": float(h*DH),
                                                     "aspect_ratio": float(w*DW)/float(h*DH),
                                                     "pose": "REAR"
                                                     }
                    }
            all_out_boxes.append(out_box)
        img_basename = os.path.basename(img_fnames_batch[ii_img])
        json_dict_res = {'img_filename': img_basename, 'bounding_box_object_annotation': all_out_boxes}
        json_fname = os.path.join(pred_json_folder, img_basename + JSON_PREFIX)
        json_file = open(json_fname, 'w')
        json.dump(json_dict_res, json_file, indent=4, sort_keys=True)
        json_file.close()
    return compute_time

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("cfg_file")
    args = parser.parse_args()

    if not os.path.isfile(args.cfg_file):
        #logger.error('Can not find configuration file : {}'.format(args.cfg_file))
        exit(-1)

    if os.path.isfile(args.cfg_file):
        mc = get_model_config(args.cfg_file)

    if tf.gfile.Exists(mc.train.LOG_TRAIN_DIR):
        tf.gfile.DeleteRecursively(mc.train.LOG_TRAIN_DIR)
    tf.gfile.MakeDirs(mc.train.LOG_TRAIN_DIR)

    if tf.gfile.Exists(mc.test.LOG_TEST_DIR):
        tf.gfile.DeleteRecursively(mc.test.LOG_TEST_DIR)
    tf.gfile.MakeDirs(mc.test.LOG_TEST_DIR)

    if tf.gfile.Exists(mc.train.CHECKPOINT_DIR):
        tf.gfile.DeleteRecursively(mc.train.CHECKPOINT_DIR)
    tf.gfile.MakeDirs(mc.train.CHECKPOINT_DIR)

    res = train(mc)

    exit(res)


from requests.utils import requote_uri
import requests
import time
import hashlib
import macDb
import binascii

unknown="unknown"

def init(dbPath):
    macDb.init(dbPath)

def getMacManufacturerId(mac):
    if(mac != None and len(mac) >8):
        return mac[:8]
    else:
        return None

def hashedMac(mac):
    if(mac != None and mac != ""):
        m = hashlib.sha256()
        m.update(str.encode(mac, encoding="UTF-8"))
        digest = m.digest()
        hashStr = binascii.b2a_base64(digest, newline=False)
        return hashStr.decode("UTF-8")
    else:
        return None

def lookupMacRequest(mac):
    url = requote_uri("https://api.macvendors.com/{}".format(mac))
    r = requests.get(url)
    # just to avoid blacklisting
    time.sleep(1)
    if (r.status_code < 300):
        return r.text
    else:
        return unknown

def lookupMac(mac):
    macId = getMacManufacturerId(mac)
    dbResult = macDb.getManufacturer(macId)
    if(dbResult == None):
        name = lookupMacRequest(mac)
        if(name != unknown):
            macDb.store(macId, name)
        return name
    else:
        print("cache hit")
        return dbResult


# -*- coding: utf-8 -*-
# __author__ = 'qinjincheng'

import numpy as np
import matplotlib.pyplot as plt
from collections import Counter

X_train = np.array([
    [158, 64],
    [170, 86],
    [183, 84],
    [191, 80],
    [155, 49],
    [163, 59],
    [180, 67],
    [158, 54],
    [170, 67],
])
print('explanatory variable: {}'.format(X_train))
y_train = ['male'] * 4 + ['female'] * 5
y_train = np.array(y_train)
print('response variable: {}'.format(y_train))

plt.figure()
plt.title('Human Heights and Weights by sex')
plt.xlabel('Height in cm')
plt.ylabel('Height in kg')
for i, x in enumerate(X_train):
    plt.scatter(x[0], x[1], c='k', marker='x' if y_train[i] == 'male' else 'D')
plt.grid(True)
plt.show()

x = np.array([[155, 70]])
distances = np.sqrt(np.sum((X_train - x) ** 2, axis=1))
print('distances: {}'.format(distances))

nearest_neighbor_indices = distances.argsort()[:3]
print('nearest neighbor indices: {}'.format(nearest_neighbor_indices))
nearest_neighbor_genders = np.take(y_train, nearest_neighbor_indices)
print('nearest neighbor genders: {}'.format(nearest_neighbor_genders))

b = Counter(nearest_neighbor_genders)
print('predict gender: {}'.format(b.most_common(1)))
print('predict gender: {}'.format(b.most_common(1)[0][0]))


import torch.utils.data as data
from torch.utils.data.dataset import Subset
from torchvision.datasets.folder import default_loader
from torchvision.transforms import functional as F
from PIL import Image
import os
import h5py
import numpy as np
import cv2
from scipy.io import loadmat
import torch

from PIL import Image

def get_dataset(config, data_transforms=None, test=False, limit=None):

    nyu_hdf5_loader = NYU_HDF5_Loader(config.data_dir, limit=limit, no_val=config.no_val)

    if test:
        datasets = {'test': NYU_Depth_V2(nyu_hdf5_loader,
                                         'test',
                                         transform = data_transforms['eval'],
                                         get_image_transform = data_transforms['eval'])
        }

    else:
        datasets = {'train': NYU_Depth_V2(nyu_hdf5_loader,
                                          'train',
                                          transform = data_transforms['train']['data'],
                                          target_transform = data_transforms['train']['target'],
                                          get_image_transform = data_transforms['eval']),
                    'val': NYU_Depth_V2(nyu_hdf5_loader,
                                        'val',
                                        transform = data_transforms['eval'],
                                        get_image_transform = data_transforms['eval'])
        }

    return datasets

detection_label_names_subset = ['BACKGROUND', 'bathtub', 'bed', 'bookshelf', 'box', 'chair', 'counter',
                      'desk', 'door', 'dresser', 'garbage bin', 'lamp', 'monitor',
                      'night stand', 'pillow', 'sink', 'sofa', 'table',
                      'television', 'toilet']

rock_val_scenes = ['bathroom_0039','bathroom_0041','bathroom_0049','bathroom_0051', 'bedroom_0098', 'bedroom_0104','bedroom_0106',
                   'bedroom_0107','bedroom_0113','bedroom_0116','bedroom_0118','bedroom_0120','bedroom_0124','bedroom_0125','bedroom_0126',
                   'bedroom_0129','bedroom_0130','classroom_0016','classroom_0018','conference_room_0002','dining_room_0028',
                   'dining_room_0031','dining_room_0034','home_office_0006','kitchen_0049','kitchen_0050','kitchen_0051','kitchen_0052',
                   'living_room_0062','living_room_0063','living_room_0068','living_room_0069','living_room_0070','living_room_0071',
                   'living_room_0078','office_0024','office_0026','playroom_0002','reception_room_0004','study_0006','study_room_0004']

all_scene_types = ['foyer', 'study', 'printer_room', 'home_storage', 'indoor_balcony', 'living_room', 'kitchen', 'computer_lab',
                   'bookstore', 'conference_room', 'laundry_room', 'reception_room', 'classroom', 'office', 'dining_room',
                   'basement', 'bedroom', 'bathroom', 'furniture_store', 'dinette', 'student_lounge', 'excercise_room',
                   'office_kitchen', 'study_room', 'playroom', 'cafe', 'home_office']

def get_bboxes_from_masks(instance_masks):
    bboxes = np.zeros((instance_masks.shape[2], 4), dtype=np.float32)
    #print(instance_masks.shape[0]) 480
    for ii in range(instance_masks.shape[2]):
        for i in range(instance_masks.shape[1]):
            if np.sum(instance_masks[:,i,ii]):
                bboxes[ii,0] = i # xmin
                break
        for i in reversed(range(instance_masks.shape[1])):
            if np.sum(instance_masks[:,i,ii]):
                bboxes[ii,2] = i # xmax
                assert bboxes[ii,2] > bboxes[ii,0], 'invalid width bbox'
                break
        for i in range(instance_masks.shape[0]):
            if np.sum(instance_masks[i,:,ii]):
                bboxes[ii,1] = i # ymin
                break
        for i in reversed(range(instance_masks.shape[0])):
            if np.sum(instance_masks[i,:,ii]):
                bboxes[ii,3] = i # ymax
                assert bboxes[ii,3] > bboxes[ii,1], 'invalid height bbox'
                break
    return bboxes

def get_instance_masks(img_object_labels, img_instances):

    H, W = img_object_labels.shape

    A = np.asarray([img_object_labels.reshape(-1,1), img_instances.reshape(-1,1)], dtype=np.uint16)
    pairs = np.unique(A, axis=1)
    pairs = pairs[:,np.sum(pairs, axis=0)!=0].transpose()

    N = pairs.shape[0]

    instance_masks = np.zeros((H,W,N))
    instance_labels = np.zeros((N,), dtype=np.int64)

    for ii in range(N):
        instance_masks[:,:,ii] = (img_object_labels == pairs[ii,0]) * (img_instances == pairs[ii,1])
        instance_labels[ii] = pairs[ii,0]
    return instance_masks, instance_labels

def sn_normalize(depthImg):
    mean = np.mean(depthImg)
    std = np.std(depthImg)
    out = depthImg.astype(np.float32) - mean
    out /= std
    out = np.clip(out, -1.5, 1.5)
    out = (out + 1.5) / 3.0
    return out

def compute_normals(depths, verbose=False):

    shape = list(depths.shape)
    shape.append(3)
    normals = np.zeros(tuple(shape))

    for i in range(depths.shape[0]):
        if verbose:
            print('Computing %d' % i)
        d_im = depths[i,:,:]
        surf = np.zeros((d_im.shape[0], d_im.shape[1], 3), dtype=np.float32)
        n = np.zeros(3, dtype=np.float32)

        img = np.array(d_im, dtype=np.float32)
        img = cv2.bilateralFilter(img, 25, 10, 10)
        img = cv2.copyMakeBorder(img, 1, 1, 1, 1, cv2.BORDER_REPLICATE)

        for x in range(0, d_im.shape[0]-1):
            for y in range(0, d_im.shape[1]-1):
                dzdx = (img[x+1,y] - img[x-1,y])/2.0
                dzdy = (img[x,y+1] - img[x,y-1])/2.0
                n[0]=-dzdx
                n[1]=-dzdy
                n[2]=1.0
                n /= np.linalg.norm(n)
                surf[x,y,0]=n.item(0)
                surf[x,y,1]=n.item(1)
                surf[x,y,2]=n.item(2)

        gauss = cv2.GaussianBlur(surf,(0,0),3)
        unsharp = cv2.addWeighted(surf,1.5,gauss,-0.5,0,surf)

        surf = sn_normalize(surf)

        normals[i,:,:,:] = surf

    return normals

class NYU_HDF5_Loader():
    """This is the HDF5 file reader for the NYU V2 Dataset

    It reads the hdf5 files and computes the indexes for the train, validation and test samples.
    It also implements the load_split function to load data in memory (used to pass it to the
    NYU_Depth_V2 class).
    """

    def __init__(self, root, limit=None, use_rock_val_scenes=True, no_val=0):
        # Begin by opening hdf5 file
        self.root = root
        self.limit = limit
        self.use_rock_val_scenes = use_rock_val_scenes
        self.no_val = no_val
        self.hdf5_dataset = h5py.File(os.path.join(self.root, 'nyu_depth_v2_labeled.mat'))
        # Open splits file and compute train - val - test splits indexes
        self.splits = loadmat(os.path.join(self.root, 'splits.mat'))
        self.split_idxs = {
            'train': self.splits['trainNdxs'].reshape(-1) - 1, # Matlab indexes start from 1
            'test': self.splits['testNdxs'].reshape(-1) - 1    # Matlab indexes start from 1 :)
        }
        self.objs = [obj for obj in self.hdf5_dataset['scenes'][0][self.split_idxs['train']]]
        self.scenes = [''.join(chr(c) for c in self.hdf5_dataset[obj]) for obj in self.hdf5_dataset['scenes'][0]]
        self.train_scenes = [''.join(chr(c) for c in self.hdf5_dataset[obj]) for obj in self.hdf5_dataset['scenes'][0][self.split_idxs['train']]]
        self.trainval_idxs = self._compute_trainval_idxs()
        self.idxs_dict = {
            'train': self.trainval_idxs[0],
            'val': self.trainval_idxs[1],
            'test': self.split_idxs['test'].tolist()
        }

    def _compute_trainval_idxs(self):
        scenes = list(set(self.train_scenes))

        train_scenes, val_scenes = set(), set()

        if self.no_val:
            print('Using the entire trainval set for training')
            train_idxs = self.split_idxs['train'].tolist()
            val_idxs = self.split_idxs['test'].tolist()
            return train_idxs, val_idxs

        elif self.use_rock_val_scenes:
            print('computing rock train-val splits using val scenes used in ROCK')
            for scene in scenes:
                if scene in rock_val_scenes:
                    val_scenes.add(scene)
                else:
                    train_scenes.add(scene)
        else:
            print('Computing random trainval set')
            for i in range(len(scenes)):
                if i % 6 == 0:
                    val_scenes.add(scenes[i])
                else:
                    train_scenes.add(scenes[i])

        train_idxs, val_idxs = [], []
        for i in self.split_idxs['train']:
            if self.scenes[i] in train_scenes:
                train_idxs.append(i)
            else:
                val_idxs.append(i)
        return train_idxs, val_idxs

    def load_split(self, phase='train', use_precomputed=True):

        assert phase in ['train', 'val', 'test']
        idxs = self.idxs_dict[phase][:self.limit]

        print('Loading NYU_Depth_V2 %s data in memory' % phase)
        images = self.hdf5_dataset['images'][idxs][()].transpose((0,3,2,1))
        labels = self.hdf5_dataset['labels'][idxs][()].transpose((0,2,1))
        instances = self.hdf5_dataset['instances'][idxs][()].transpose((0,2,1))
        depths = self.hdf5_dataset['depths'][idxs][()].transpose((0,2,1))

        if use_precomputed:
            normals = np.load('./data/normals.npy')[idxs]
        else:
            print('Calculating Surface Normals from Depth Maps')
            normals = compute_normals(depths * 25.5)

        names = [''.join(chr(c) for c in self.hdf5_dataset[obj]) for obj in self.hdf5_dataset['names'][0]]
        scene_types = [''.join(chr(c) for c in self.hdf5_dataset[obj]) for obj in self.hdf5_dataset['sceneTypes'][0][idxs]]
        scene_idxs = [all_scene_types.index(scene_type) for scene_type in scene_types]

        return idxs, images, labels, instances, depths, names, scene_idxs, normals

class NYU_Depth_V2(data.Dataset):

    def __init__(self, hdf5_loader, phase, transform=None, depth_transform=None, target_transform=None, get_image_transform=None,
                 detection=True, use_label_subset=True, remove_negatives_for_detection=True, compute_gaussian_masks=False):
        # init path and transforms
        self.transform = transform
        self.depth_transform = depth_transform
        self.target_transform = target_transform
        self.get_image_transform = get_image_transform
        self.compute_gaussian_masks = compute_gaussian_masks
        # Load data from loader
        data = hdf5_loader.load_split(phase)
        # Pointers to data
        self.ids = data[0]
        self.images = data[1]
        self.labels = data[2]
        self.instances = data[3]
        self.depths = data[4]
        self.names = data[5]
        self.scene_types = data[6]
        self.normals = data[7]
        # Logic to use only the provided subset labels
        self.use_label_subset = use_label_subset
        if use_label_subset:
            self.class_names = detection_label_names_subset
            self.labels_subset = [136, 157, 88, 26, 5, 7, 36, 28, 169, 12, 144, 49, 158, 119, 24, 83, 19, 172, 124]
            self.labels_to_subset_labels = {label: i+1 for i, label in enumerate(self.labels_subset)}
            self._use_label_subset()

        self.detection = detection
        if detection:
            self.bboxes, self.bboxes_labels = self._compute_bboxes()
            if remove_negatives_for_detection:
                to_remove_idxs = []
                for i in range(len(self.bboxes)):
                    if len(self.bboxes[i]) == 0:
                        to_remove_idxs.append(i)
                if len(to_remove_idxs):
                    self._remove_negatives(to_remove_idxs)

    def _remove_negatives(self, idxs):
        print('Removing %d samples without bboxes' % len(idxs))
        to_keep = [i for i in range(self.images.shape[0]) if i not in idxs]
        self.ids = [self.ids[i] for i in range(self.images.shape[0]) if i not in idxs]
        self.images = self.images[to_keep]
        self.labels = self.labels[to_keep]
        self.instances = self.instances[to_keep]
        self.depths = self.depths[to_keep]
        self.normals = self.normals[to_keep]
        self.scene_types = [self.scene_types[i] for i in range(len(self.scene_types)) if i not in idxs]
        self.bboxes = [self.bboxes[i] for i in range(len(self.bboxes)) if i not in idxs]
        self.bboxes_labels = [self.bboxes_labels[i] for i in range(len(self.bboxes_labels)) if i not in idxs]

    def _use_label_subset(self):
        print('Converting labels to only use the 19 classes subset')
        # create a mask for valid labels
        mask = np.isin(self.labels, self.labels_subset).astype(np.uint32)
        # use the mask to update instances and labels to use the 19 classes (everything else is set to zero)
        self.labels = self.labels * mask
        self.instances = self.instances * mask
        del mask

    def _compute_gaussian_masks(self):
        masks = np.zeros(self.depths.shape)

    def _compute_bboxes(self):
        bboxes = []
        bboxes_labels = []
        for i in range(self.images.shape[0]):
            instance_masks, instance_labels = get_instance_masks(self.labels[i], self.instances[i])
            img_bboxes = get_bboxes_from_masks(instance_masks)
            if self.use_label_subset:
                for ii in range(instance_labels.shape[0]):
                    instance_labels[ii] = self.labels_to_subset_labels[instance_labels[ii]]
            bboxes.append(img_bboxes)
            bboxes_labels.append(instance_labels)
        return bboxes, bboxes_labels

    def __getitem__(self, index):

        # access to indexed elements
        img, label, instances, depth, normals = self.images[index], self.labels[index], self.instances[index], self.depths[index], self.normals[index]
        scene_type = self.scene_types[index]

        bboxes = self.bboxes[index]
        bboxes_labels = self.bboxes_labels[index]

        # Load image
        img = Image.fromarray(img, mode='RGB')

        # Load labels, depth and instances as 1-channel images
        label = Image.fromarray(label.astype(np.uint32), mode='I')
        instances = Image.fromarray(instances, mode='P')
        depth = Image.fromarray(depth, mode='F')

        # Load normals
        normals_0 = Image.fromarray(normals[:,:,0].astype(np.float32), mode='F')
        normals_1 = Image.fromarray(normals[:,:,1].astype(np.float32), mode='F')
        normals_2 = Image.fromarray(normals[:,:,2].astype(np.float32), mode='F')

        normals = (normals_0, normals_1, normals_2)

        # Apply transforms
        try:
            if self.transform is not None:
                img, bboxes, bboxes_labels, depth, normals = self.transform(img, bboxes, bboxes_labels, depth, normals)
                normals = torch.stack(list(normals), 1).squeeze()
        except ValueError:
            print('Error at index %d' % index)
            print(bboxes)
            raise Exception
        if self.target_transform is not None:
            bboxes, bboxes_labels = self.target_transform(bboxes, bboxes_labels)

        # return transformed data and labels
        return index, img, depth, scene_type, bboxes, bboxes_labels, normals

    def __len__(self):
        return self.images.shape[0]

    def get_image(self, index):
        image = self.images[index]
        image = Image.fromarray(image, mode='RGB')
        if self.get_image_transform is not None:
            image, _, _, _, _ = self.get_image_transform(image)
        return image

    def get_annotation(self, index):
        '''This method is to conform with the ssd implementation of qfgaohao'''
        bboxes, bboxes_labels = self.bboxes[index], self.bboxes_labels[index]
        is_difficult = np.zeros(bboxes.shape[0], dtype=np.uint8)
        annotation = (bboxes, bboxes_labels, is_difficult)
        return self.ids[index], annotation