__author__ = 'linh'
import numpy as np
import tensorflow as tf
import scipy.io
import matplotlib.pyplot as plt
from cf_dae import cf_vae_extend, params
from scipy.sparse import load_npz
import  argparse
import os
import csv
import pickle

np.random.seed(0)
tf.set_random_seed(0)

parser = argparse.ArgumentParser(description='Process some integers.')
parser.add_argument('--model',  type=int, default=0,
                   help='type of model: 0-only text, 1-text+image, 2-text+image+structure, 3-text+structure')
parser.add_argument('--ckpt_folder',  type=str, default='pre_model/exp1/',
                   help='where model is stored')
parser.add_argument('--initial',  type=bool, default=True,
                   help='where model is stored')
parser.add_argument('--iter',  type=int, default=30,
                   help='where model is stored')
parser.add_argument('--data_dir',  type=str, default='data/amazon',
                   help='where model is stored')
parser.add_argument('--zdim',  type=int, default=50,
                   help='where model is stored')
parser.add_argument('--gridsearch',  type=int, default=0,
                   help='gridsearch or not')
parser.add_argument('--mat_file',  type=int, default=0,
                   help='gridsearch or not')
parser.add_argument('--user_no',  type=int, default=6040,
                   help='gridsearch or not')
parser.add_argument('--item_no',  type=int, default=3883,
                   help='gridsearch or not')
args = parser.parse_args()
model_type = args.model
ckpt = args.ckpt_folder
initial = args.initial
iter = args.iter
data_dir = args.data_dir
zdim = args.zdim
gs = args.gridsearch
print(model_type)

def load_cvae_data(data_dir):

  f = open(os.path.join(data_dir,"dataset.pkl"), 'rb')
  dataset = pickle.load(f)
  variables = load_npz(os.path.join(data_dir, "mult_nor.npz"))
  dataset["content"] = variables.toarray()

  train_item = [0] * dataset['item_no']
  train_tag = [0] * dataset['tag_no']
  tag_list = np.where(dataset['tag_item_onehot'] == 1)
  for i in range(len(tag_list[0])):
      item_id = tag_list[0][i]
      tag_id = tag_list[1][i]
      if train_item[item_id] == 0:
          train_item[item_id] = [tag_id]
      else:
          train_item[item_id].append(tag_id)

      if train_tag[tag_id] == 0:
          train_tag[tag_id] = [item_id]
      else:
          train_tag[tag_id].append(item_id)
  for i in range(len(train_item)):
      if train_item[i] == 0:
          train_item[i] = []
  for i in range(len(train_tag)):
      if train_tag[i] == 0:
          train_tag[i] = []

  dataset["train_users"] = train_item
  dataset["train_items"] = train_tag

  test_tag_id = []
  test_tag_y = []

  # min_len = min(len(dataset['test']), len(dataset['tag_test']))
  #
  # for i in range(min_len):
  #     try:
  #         idx = test_tag_id.index(dataset['test'][i, 1])
  #         test_tag_y[idx] += dataset['tag_test'][i]
  #         print(test_tag_y[idx])
  #     except:
  #         test_tag_id.append(dataset['test'][i, 1])
  #         test_tag_y.append(dataset['tag_test'][i])
  # dataset["test_item_id"] = test_tag_id
  # dataset["test_item_tag"] = test_tag_y
  return dataset

def load_rating(path):
  arr = []
  for line in open(path):
    a = line.strip().split()
    if a == []:
      l = []
    else:
      l = [int(x) for x in a[1:]]
    arr.append(l)
  return arr

params = params()
params.lambda_u = 10
params.lambda_v = 1
params.lambda_r = 10
params.C_a = 1
params.C_b = 0.01
params.max_iter_m = 1
params.EM_iter = args.iter
params.num_iter = 150

C = [0.1, 1, 10]

# # for updating W and b in vae
# self.learning_rate = 0.001
# self.batch_size = 500
# self.num_iter = 3000
# self.EM_iter = 100

data = load_cvae_data(data_dir)
np.random.seed(0)
tf.set_random_seed(0)

# images = np.fromfile(os.path.join(data_dir,"images.bin"), dtype=np.uint8)
# img = images.reshape((13791, 32, 32, 3))
# img = img.astype(np.float32)/255
num_factors = 50
best_recall = 0
best_hyper = []
dim = data['content'].shape[1]
train_test = [data["train_users"][i] for i in data['tag_test'].keys()]

if gs == 1:
    i = 0
    recalls = []
    for u in [0.1, 1, 10]:
        params.lambda_u = u
        for v in [1, 10, 100]:
            params.lambda_v = v
            for r in [0.1, 1, 10]:
                params.lambda_r = r
                if i > -1:
                    model = cf_vae_extend(num_users=data['item_no'], num_items=data['tag_no'], num_factors=num_factors,
                                          params=params,input_dim=dim, encoding_dims=[400, 200], z_dim=zdim,
                                          decoding_dims=[200,400,dim], decoding_dims_str=[200, 4526], loss_type='cross_entropy',
                                          model = model_type, ckpt_folder=ckpt)
                    model.fit(data["train_users"], data["train_items"], data["content"], params, data["tag_test"].keys())
                    model.save_model(os.path.join(ckpt,"cf_dae_%d_%d.mat"%(model_type, i)))
                    # model.load_model("cf_vae.mat")
                    f = open(os.path.join(ckpt, "result_cdae_%d.txt"%model_type), 'a')
                    f.write("%d-----------%f----------%f----------%f\n"%(i,u,v,r))
                    pred_all = model.predict_all()
                    pred_all = pred_all[data["tag_test"].keys()]

                    recall = model.predict_val(pred_all, train_test, data['tag_test'].values(), f)
                    f.write("\n")
                    f.close()
                    if recall > best_recall:
                        best_recall = recall
                        best_hyper = [u,v, r, i]

                    print(u, v, r)
                i += 1

    f = open(os.path.join(ckpt, "result_sum.txt"), "a")
    f.write("Best recall CDAE: %f at %d (%f, %f, %f)\n" % (best_recall, best_hyper[3], best_hyper[0], best_hyper[1],
                                                                                best_hyper[2]))
    f.close()
else:
    u = [0.1, 1, 10]
    v = [1,10, 100]
    r = [0.1, 1, 10]
    params.lambda_u = u[int(args.mat_file/9)]
    params.lambda_v = v[int((args.mat_file%9)/3)]
    params.lambda_r = r[int(args.mat_file%3)]

    model = cf_vae_extend(num_users=data['item_no'], num_items=data['tag_no'], num_factors=num_factors, params=params,
                          input_dim=dim, encoding_dims=[400, 200], z_dim=zdim, decoding_dims=[200, 400, dim],
                          decoding_dims_str=[200, 4526], loss_type='cross_entropy',
                          model=model_type, ckpt_folder=ckpt)
    model.fit(data["train_users"], data["train_items"], data["content"], params, data["tag_test"].keys())
    model.save_model(os.path.join(ckpt,"cf_dae_%d.mat"%(model_type)))
    #model.load_model(os.path.join(ckpt, "cf_dae_0.mat"))
    pred = model.predict_all()
    pred_all = pred[data["tag_test"].keys()]
    recall = model.predict_val(pred_all, train_test, data['tag_test'].values())


# -*- coding: utf-8 -*-
"""
# Author      : Camey
# DateTime    : 2021/12/1 7:47 下午
# Description :
"""
import sys, os
import datetime
import torch
import gym
from DQN.agent import DQN
from COMMON.utils import save_results, make_dir
from COMMON.plot import plot_rewards,plot_rewards_cn

curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径
parent_path = os.path.dirname(curr_path) # 父路径
sys.path.append(parent_path) # 添加父路径到系统路径sys.path

curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")  # 获取当前时间

class DQNConfig:
    def __init__(self):
        self.algo = "DQN"
        self.env = "CartPole-v0"
        self.result_path = curr_path + "/outputs/" + self.env + \
                           '/' + curr_time + '/results/'  # 保存结果的路径
        self.model_path = curr_path + "/outputs/" + self.env + \
                          '/' + curr_time + '/models/'  # 保存模型的路径
        self.train_eps = 200
        self.eval_eps = 50
        self.gamma = 0.95
        self.epsilon_start = 0.9 #e-greedy策略中初始的e
        self.epsilon_end = 0.01
        self.epsilon_decay = 500

        self.lr = 0.0001
        self.memory_capacity = 100000
        self.batch_size = 64
        self.target_update = 4
        self.device = torch.device(
            "cuda" if torch.cuda.is_available() else "cpu"
        )
        self.hidden_dim = 256

def env_agent_config(cfg, seed=1):
    env = gym.make(cfg.env)
    env.seed(seed)
    state_dim = env.observation_space.shape[0]
    action_dim = env.action_space.n
    agent = DQN(cfg, state_dim, action_dim)
    return env, agent

def train(cfg, env, agent):
    rewards = []
    ma_rewards = []
    for i_ep in range(cfg.train_eps):
        state = env.reset()
        ep_reward = 0
        while True:
            action = agent.choose_action(state)
            next_state, reward, done, _ = env.step(action)
            ep_reward += reward
            agent.memory.push(state, action, reward, next_state, done)
            state = next_state
            agent.update()
            if done:
                break
        if (i_ep+1) % cfg.target_update == 0:
            agent.target_net.load_state_dict(agent.policy_net.state_dict())

        if (i_ep+1)%10 == 0:
            print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.train_eps, ep_reward))
        rewards.append(ep_reward)

        if ma_rewards:
            ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward)
        else:
            ma_rewards.append(ep_reward)

    print("训练完成")
    return rewards, ma_rewards

def eval(cfg, env, agent):
    rewards = []
    ma_rewards = []
    for i_ep in range(cfg.eval_eps):
        ep_reward = 0
        state = env.reset()
        while True:
            env.render()
            action = agent.predict(state)
            next_state, reward, done, _ = env.step(action)
            state = next_state
            ep_reward += reward
            if done:
                break
        rewards.append(ep_reward)
        if ma_rewards:
            ma_rewards.append(ma_rewards[-1]*0.9 + ep_reward*0.1)
        else:
            ma_rewards.append(ep_reward)
        print(f"回合：{i_ep + 1}/{cfg.eval_eps}, 奖励：{ep_reward:.1f}")
    print('完成测试！')
    return rewards, ma_rewards

if __name__ == "__main__":
    cfg = DQNConfig()
    env, agent = env_agent_config(cfg, seed=1)
    rewards, ma_rewards = train(cfg, env, agent)
    make_dir(cfg.result_path, cfg.model_path)
    agent.save(path=cfg.model_path)
    save_results(rewards, ma_rewards, tag="train", path = cfg.result_path)
    plot_rewards_cn(rewards, ma_rewards, tag="train",
                    algo=cfg.algo, path=cfg.result_path)

    # 测试
    env, agent = env_agent_config(cfg, seed=10)
    agent.load(path=cfg.model_path)
    rewards, ma_rewards = eval(cfg, env, agent)
    save_results(rewards, ma_rewards, tag='eval', path=cfg.result_path)
    plot_rewards_cn(rewards, ma_rewards, tag="eval", env=cfg.env, algo=cfg.algo, path=cfg.result_path)


import cv2
import numpy as np
from os import listdir
from os.path import isfile, join
data_path='D:/Projects/Faces/'
onlyfiles=[f for f in listdir(data_path)if isfile(join(data_path,f))]

Training_Data,labels =[], []

for i,  files in enumerate(onlyfiles):
    image_path=data_path+onlyfiles[i]
    images=cv2.imread(image_path,cv2.IMREAD_GRAYSCALE)
    Training_Data.append(np.asarray(images,dtype=np.uint8))
    labels.append(i)

labels=np.asarray(labels,dtype=np.int32)

model=cv2.face.LBPHFaceRecognizer_create()  ## Linear Binary Phase Histogram(LBPH)
model.train(np.asarray(Training_Data),np.asarray(labels))

print("!!! Model Training Complete, Ready to Launch !!! ")

face_classifier= cv2.CascadeClassifier('C:/python37/Lib/site-packages/cv2/data/haarcascade_frontalface_default.xml')
def face_detector(image,size=0.5):
    gray=cv2.cvtColor(image,cv2.COLOR_RGB2GRAY)
    faces=face_classifier.detectMultiScale(gray,1.3,5)

    if faces is():
        return image,[]
    for(x,y,w,h) in faces:
        cv2.rectangle(image,(x,y),(x+w,y+h),(0,255,255),2)
        roi=image[y:y+h,x:x+w]
        roi=cv2.resize(roi,(200,200))

    return  image,roi

cap=cv2.VideoCapture(0)
while True:
    ret,frame=cap.read()

    image, face=face_detector(frame)

    try:

        face=cv2.cvtColor(face,cv2.COLOR_RGB2GRAY)
        result=model.predict(face)
        if result[1]<500:
            conf=int(100*(1-(result[1])/300))
            dispaly_string=str(conf)+'% Face Matches'
        cv2.putText(image,dispaly_string,(100,200),cv2.FONT_HERSHEY_DUPLEX,1,(250,120,255),2)
        if conf>70:
            cv2.putText(image,"Unloking The System", (230, 450), cv2.FONT_HERSHEY_DUPLEX, 1, (0, 255, 0), 2)
            cv2.imshow("Image Cropper",image)
        else:
            cv2.putText(image, "Invalid User, System Can not be unlocked", (230, 450), cv2.FONT_HERSHEY_DUPLEX, 1, (0, 0, 255), 2)
            cv2.imshow("Image Cropper", image)

    except:

        cv2.putText(image, "No Face Detected", (230, 450), cv2.FONT_HERSHEY_DUPLEX, 1, (255, 0, 0), 2)

        cv2.imshow("Image Cropper", image)
        pass

    if cv2.waitKey(1)==13:
        break

cap.release()

cv2.destroyAllWindows()


from functools import reduce
from math import sqrt, log10
from typing import Tuple, List
from matplotlib.pyplot import plot, show, xlabel, ylabel, title, legend
from random import randrange, random

from numpy.random import random
import seaborn as sns
import pandas as pd
import numpy as np

def lineralRegression(values: List[Tuple[float, float]]) -> Tuple[float, float]:
  n = len(values)
  sumXY = reduce(lambda acc, t: acc + (lambda x, y: x * y)(*t), values, 0.0)
  sumX = reduce(lambda acc, t: acc + (lambda x, y: x)(*t), values, 0.0)
  sumY = reduce(lambda acc, t: acc + (lambda x, y: y)(*t), values, 0.0)
  sumX2 = reduce(lambda acc, t: acc + (lambda x, y: x ** 2)(*t), values, 0.0)

  a = (n * sumXY - sumX * sumY) / (n * sumX2 - sumX ** 2)
  b = (sumY - a * sumX) / n

  return (a, b)

def M(values: List[float]) -> float:
  return sum(values) / len(values)

def multiply(set1: List[float], set2: List[float]) -> List[float]:
  return list(map(lambda t: set1[t[0]] * t[1], enumerate(set2)))

def substract(set1: List[float], set2: List[float]) -> List[float]:
  return list(map(lambda t: set1[t[0]] - t[1], enumerate(set2)))

def decrease(s: List[float], value: float) -> List[float]:
  return list(map(lambda v: v - value, s))

def pow(s: List[float], power: float) -> List[float]:
  return list(map(lambda v: v ** power, s))

def covariance(set1: List[float], set2: List[float]) -> float:
  return M(multiply(set1, set2)) - M(set1) * M(set2)

def variance(s: List[float]) -> float:
  a = pow(s, 2)
  b = M(s) ** 2
  c = M(a)
  return c - b

def deviation(s: List[float]) -> float:
  return sqrt(variance(s))

def correlation(set1: List[float], set2: List[float]) -> float:
  return covariance(set1, set2) / (deviation(set1) * deviation(set2))

def getCompensationPoint(a: float, b: float, X: List[float], Y: List[float]):
  n = len(X)

  m = sum(Y)
  s = sum(X)
  l = sum(map(lambda x: x ** 2, X))
  k = sum(map(lambda t: (lambda i, x: x * Y[i])(*t), enumerate(X)))

  x0 = (a * s + b * l - k) / (a * n + b * s - m)
  y0 = a * (n + 1) - sum(Y) + b * (sum(X) + x0)

  print("{} {} ".format(x0, y0))

  return (x0, y0)

def gaussCached():
  global secondValue
  secondValue = 0.0

  def gauss():
    global secondValue

    if secondValue != 0.0:
      v = secondValue
      secondValue = 0.0
      return v

    while True:
      u = random() * 2 - 1
      v = random() * 2 - 1

      s = u ** 2 + v ** 2

      if s <= 1 and s != 0:
        r = sqrt(-2 * log10((s)) / s)

        secondValue = v * r
        return u * r

  return gauss

def getPoints(X: List[float], Y: List[float]):
  points: List[Tuple[float, float]] = sorted(
      list(map(lambda t: (X[t[0]], t[1]), enumerate(Y))))

  return points

def test3():
  data = pd.read_csv('files/prog2_input.csv')

  a = data["a"][0]
  b = data["b"][0]

  amount = 100
  maxX = 4000
  randomizationScale = 10

  X = [randrange(0, maxX) for _ in range(amount)]
  Y = [a * x + b for x in X]

  gauss = gaussCached()
  deviations = [gauss() * randomizationScale for _ in range(amount)]

  # gN = np.random.normal(Y, 1, 10)

  YDeviated = [Y[i] + deviations[i] for i in range(amount)]
  # print(gN)
  # print(YDeviated)

  points = getPoints(X, YDeviated)
  # pointsN = getPoints(X, gN)

  (a2, b2) = lineralRegression(points)

  plot(*zip(*points), 'go', label="generated points")
  # plot(*zip(*pointsN), 'yo', label="generated points")
  plot([0, maxX], [b, a * maxX + b], 'b', label="target regression")
  plot([0, maxX], [b2, a2 * maxX + b2], 'r', label="real regression")
  legend(loc='upper left')

  pd.DataFrame(points).to_csv("files/prog2_output.csv", index=False)

  show()

def test1():
  data = pd.read_csv('files/prog1_input.csv')
  views = data['views']
  downloads = data['downloads']

  points = sorted(
      list(map(lambda t: (views[t[0]], t[1]), enumerate(downloads))))
  (a, b) = lineralRegression(points)

  print("a: {}, b: {}".format(a, b))

  var = variance(views)
  dev = deviation(views)
  r = correlation(views, downloads)
  average = M(views)

  devN = np.std(views)
  varN = np.var(views)
  rN = np.corrcoef(views, downloads)

  print("devN: {}".format(devN))
  print("varN: {}".format(varN))
  print("rN: {}".format(rN))

  print("variance: {}".format(var))
  print("deviation: {}".format(dev))
  print("correlation: {}".format(r))
  print("average: {}".format(average))

  outputData = {
      "Variance": [var],
      "Deviation": [dev],
      "Correlation": [r],
      "Average": [average]
  }

  pd.DataFrame(outputData).to_csv("files/prog1_output.csv", index=False)

  xxx = list(map(lambda t: t[0], points))
  yyy = list(map(lambda t: t[1], points))

  regressionPointsX = [0, 10000]
  regressionPointsY = list(map(lambda x: x * a + b, regressionPointsX))

  plot(xxx, yyy, 'go', label="data poitns")
  plot(regressionPointsX, regressionPointsY, 'r', label="lineral regression")
  legend(loc='upper left')
  title('Task 1')
  xlabel('views')
  ylabel('downloads')

  # sns.lineplot(data=pd.DataFrame(
  #     {'x': regressionPointsX, 'y': regressionPointsY}))
  # sns.scatterplot(data=pd.DataFrame([xxx, yyy]))

  show()

if __name__ == '__main__':
  test1()


# -*- coding: utf-8 -*-

from Acquisition import aq_get
from collective.frontpage.interfaces import ICollectiveFrontpageLayer
from plone.app.contenttypes.testing import PLONE_APP_CONTENTTYPES_FIXTURE
from plone.app.robotframework.testing import REMOTE_LIBRARY_BUNDLE_FIXTURE
from plone.app.testing import applyProfile
from plone.app.testing import FunctionalTesting
from plone.app.testing import IntegrationTesting
from plone.app.testing import PloneSandboxLayer
from plone.app.testing import SITE_OWNER_NAME
from plone.app.testing import SITE_OWNER_PASSWORD
from plone.testing import z2
from zope.interface import alsoProvides

import collective.frontpage
import collective.sidebar
import plonetheme.tokyo

def set_browserlayer(request):
    """
    Set the BrowserLayer for the request.
    We have to set the browserlayer manually, since importing the profile alone
    doesn't do it in tests.
    """
    alsoProvides(request, ICollectiveFrontpageLayer)

def setup_sdm(portal):
    """
    Setup session data manager.
    """
    tf_name = "temp_folder"
    idmgr_name = "browser_id_manager"
    toc_name = "temp_transient_container"
    sdm_name = "session_data_manager"

    from Products.Sessions.BrowserIdManager import BrowserIdManager
    from Products.Sessions.SessionDataManager import SessionDataManager
    from Products.TemporaryFolder.TemporaryFolder import MountedTemporaryFolder
    from Products.Transience.Transience import TransientObjectContainer
    import transaction

    bidmgr = BrowserIdManager(idmgr_name)
    tf = MountedTemporaryFolder(tf_name, title="Temporary Folder")
    toc = TransientObjectContainer(
        toc_name, title="Temporary Transient Object Container", timeout_mins=20
    )
    session_data_manager = SessionDataManager(
        id=sdm_name,
        path=tf_name + "/" + toc_name,
        title="Session Data Manager",
        requestName="TESTOFSESSION",
    )
    portal._setObject(idmgr_name, bidmgr)
    portal._setObject(sdm_name, session_data_manager)
    portal._setObject(tf_name, tf)
    portal.temp_folder._setObject(toc_name, toc)
    transaction.commit()

class CollectiveFrontpageLayer(PloneSandboxLayer):

    defaultBases = (PLONE_APP_CONTENTTYPES_FIXTURE,)

    def setUpZope(self, app, configurationContext):
        request = aq_get(app, "REQUEST")
        request.environ["HTTP_ACCEPT_LANGUAGE"] = "de"
        self.loadZCML(package=collective.frontpage)
        self.loadZCML(package=collective.sidebar)
        self.loadZCML(package=plonetheme.tokyo)

    def setUpPloneSite(self, portal):
        applyProfile(portal, "collective.frontpage:default")
        portal.acl_users.userFolderAddUser(
            SITE_OWNER_NAME, SITE_OWNER_PASSWORD, ["Manager"], []
        )

class CollectiveFrontpageTestingLayer(CollectiveFrontpageLayer):
    def setUpPloneSite(self, portal):  # noqa
        applyProfile(portal, "collective.frontpage:demo")
        portal.acl_users.userFolderAddUser(
            SITE_OWNER_NAME, SITE_OWNER_PASSWORD, ["Manager"], []
        )

class CollectiveFrontpageSessionLayer(CollectiveFrontpageTestingLayer):
    def setUpPloneSite(self, portal):
        super(CollectiveFrontpageSessionLayer, self).setUpPloneSite(portal)
        setup_sdm(portal)

COLLECTIVE_FRONTPAGE_TESTING_FIXTURE = CollectiveFrontpageTestingLayer()
COLLECTIVE_FRONTPAGE_ACCEPTANCE_SESSION_FIXTURE = CollectiveFrontpageSessionLayer()

COLLECTIVE_FRONTPAGE_INTEGRATION_TESTING = IntegrationTesting(
    bases=(COLLECTIVE_FRONTPAGE_TESTING_FIXTURE,),
    name="CollectiveFrontpageLayer:IntegrationTesting",
)

COLLECTIVE_FRONTPAGE_FUNCTIONAL_TESTING = FunctionalTesting(
    bases=(COLLECTIVE_FRONTPAGE_TESTING_FIXTURE,),
    name="CollectiveFrontpageLayer:FunctionalTesting",
)

COLLECTIVE_FRONTPAGE_ACCEPTANCE_TESTING = FunctionalTesting(
    bases=(
        COLLECTIVE_FRONTPAGE_ACCEPTANCE_SESSION_FIXTURE,
        REMOTE_LIBRARY_BUNDLE_FIXTURE,
        z2.ZSERVER_FIXTURE,
    ),
    name="CollectiveFrontpageLayer:AcceptanceTesting",
)


from setuptools import find_packages, setup

with open('requirements.txt') as f:
    required = f.read().splitlines()

setup(name='pig_latin_microservice',
      version='1.0',
      description='Vicarious Pig Latin Microservice Project',
      author='Christopher Tham',
      platforms=['any'],
      license='MIT',
      packages=find_packages(),
      install_requires=required)


import argparse
import asyncio
import json
import sys
from typing import List
from datetime import datetime
import pathlib

from pixivpy_async import AppPixivAPI
from pixivpy_async.utils import JsonDict

from bookmark_classify import consts, get_bookmarks, bookmark_classify
from bookmark_classify.utils import print_override

config_path = "config.json"
RESTRICT_ALL = "all"

BOOKMARKS_PUBLIC_PATH = "bookmarks_public.json"
BOOKMARKS_PRIVATE_PATH = "bookmarks_private.json"

parser = argparse.ArgumentParser(description="ブックマークを整理します。")
parser.add_argument("-r", "--restrict", default=RESTRICT_ALL,
                    choices=[RESTRICT_ALL, consts.RESTRICT_PUBLIC, consts.RESTRICT_PRIVATE],
                    help="整理するブックマークのプライバシー設定を指定します。"
                    "'public'の場合は公開、'private'の場合は非公開、'all'の場合は両方が対象になります。"
                    "デフォルトは`%(default)s`です。")
parser.add_argument("-ou", "--only-uncategorized", action="store_true",
                    help="未分類のみを対象にするかを指定します。このオプションはフラグです。")
parser.add_argument("-gb", "--get-bookmarks", action="store_true",
                    help="キャッシュの有無に関わらず、ブックマークを取得するかを指定します。このオプションはフラグです。")
parser.add_argument("-cp", "--config-path", default=config_path, type=pathlib.Path,
                    help="configファイルのパスを指定します。指定しない場合、`%(default)s`から読み込まれます。")

class Config():
    def __init__(
        self,
        refresh_token: str,
        progress_public: int,
        progress_private: int,
        bookmarks_public: str,
        bookmarks_private: str,
        exclude_tags: List[str],
        private_tags: List[str],
        preferred_tags: str,
        delete_tags: List[str],
        delete_if_unknown: bool
    ) -> None:
        self.refresh_token = refresh_token or ""
        self.progress_public = progress_public or -1
        self.progress_private = progress_private or -1
        self.bookmarks_public = bookmarks_public
        self.bookmarks_private = bookmarks_private
        self.exclude_tags = exclude_tags
        self.private_tags = private_tags
        self.preferred_tags = preferred_tags or "bookmark"
        self.delete_tags = delete_tags
        self.delete_if_unknown = delete_if_unknown or False

        if self.progress_public < 0:
            self.progress_public = -1
        if self.progress_private < 0:
            self.progress_private = -1

    @staticmethod
    def from_jsonfile() -> "Config":
        with open(config_path, "r", encoding="utf-8") as f:
            config: "Config" = json.load(f, object_hook=lambda x: Config(**x))
            return config

    def to_jsonfile(self) -> None:
        with open(config_path, "w", encoding="utf-8") as f:
            json.dump(self.__dict__, f, ensure_ascii=False, indent=4)

async def classify(api: AppPixivAPI, config: Config, args):

    if args.only_uncategorized:
        bookmark_tag = "未分類"
    else:
        bookmark_tag = None

    async def _classify(restrict):
        if restrict == consts.RESTRICT_PRIVATE:
            cache_path = config.bookmarks_private
            new_cache_path = BOOKMARKS_PRIVATE_PATH
            progress = config.progress_private
        else:
            cache_path = config.bookmarks_public
            new_cache_path = BOOKMARKS_PUBLIC_PATH
            progress = config.progress_public

        if not args.get_bookmarks and cache_path:
            try:
                print_override("ブックマークのキャッシュを読み込み中です...")
                with open(cache_path, "r", encoding="utf-8") as f:
                    bookmarks = json.load(f, object_hook=JsonDict)
                    should_get_bookmarks = False
            except Exception:
                should_get_bookmarks = True
            else:
                print_override("ブックマークのキャッシュを読み込みました。")
        else:
            should_get_bookmarks = True

        if should_get_bookmarks:
            if args.get_bookmarks:
                print_override("--get-bookmarksフラグが有効なため、ブックマークを取得しています...")
            else:
                print_override("ブックマークのキャッシュが読み込めなかったため、取得しています...")
            bookmarks = await get_bookmarks.get_all_bookmarks_illust(
                api,
                api.user_id,
                restrict, bookmark_tag)
            with open(new_cache_path, "w", encoding="utf-8") as f:
                json.dump(bookmarks, f, ensure_ascii=False, indent=4)

            if restrict == consts.RESTRICT_PRIVATE:
                config.bookmarks_private = new_cache_path
                config.progress_private = -1
            else:
                config.bookmarks_public = new_cache_path
                config.progress_public = -1

        bookmarks.reverse()
        bookmarks = bookmarks[progress + 1:]

        config.to_jsonfile()
        print_override("ブックマークを取得しました。")

        bookmarks_len = len(bookmarks)

        async def on_success(index, _):
            if restrict == consts.RESTRICT_PRIVATE:
                config.progress_private = progress + 1 + index
            else:
                config.progress_public = progress + 1 + index
            print_override(f"進捗: {restrict} - {index + 1} / {bookmarks_len}")

        async def on_ratelimited(index, illust, ratelimited):
            config.to_jsonfile()
            t_now = datetime.now().time()
            print_override(f"{ratelimited} date: {t_now}")
            await asyncio.sleep(600)

        await bookmark_classify.bookmarks_classify(
            api,
            bookmarks,
            config.exclude_tags,
            config.private_tags,
            config.preferred_tags,
            config.delete_tags,
            config.delete_if_unknown,
            on_success=on_success,
            on_ratelimited=on_ratelimited
            )

        config.to_jsonfile()
        print_override(f"進捗: {restrict} - 終了")

    if args.restrict in (RESTRICT_ALL, consts.RESTRICT_PUBLIC):
        await _classify(consts.RESTRICT_PUBLIC)
    if args.restrict in (RESTRICT_ALL, consts.RESTRICT_PRIVATE):
        await _classify(consts.RESTRICT_PRIVATE)

    config.to_jsonfile()
    print_override("ブックマークの整理が終了しました。")

async def _login(api: AppPixivAPI, config: Config):
    try:
        await api.login(refresh_token=config.refresh_token)
    except Exception as e:
        print_override("ログインできませんでした。リフレッシュトークンが正しいか確認してください。")
        print("\n例外情報を以下に示します。:")
        print(e)
        sys.exit(1)

async def _main(api: AppPixivAPI, config: Config, args):
    print_override("ログイン中...")
    await _login(api, config)
    print_override("ログインが完了しました。")
    await asyncio.gather(classify(api, config, args))

def main(args):
    if args.config_path:
        global config_path
        config_path = args.config_path

    try:
        print("config.jsonを読み込んでいます...", end="")
        config = Config.from_jsonfile()
        asyncio.run(_main(AppPixivAPI(), config, args))
    except KeyboardInterrupt:
        config.to_jsonfile()
        print(
            "\n進捗を保存し、処理を中断しました\n進捗:\n"
            f"公開 - {config.progress_public + 1}\n"
            f"非公開- {config.progress_private + 1}")
        sys.exit(0)

if __name__ == "__main__":
    args = parser.parse_args()
    main(args)


"""Model module.

This module manages all the logic of the program.
"""

import os
import tempfile
from copy import deepcopy
import itertools
from typing import TYPE_CHECKING, Any

import nltk

from config import DATA_DIR, PATTERNS_FILE, SETTINGS_FILE
import model.file_op as fo
from model.settings import Settings
from model.tools.string_block import StringBlock, StringBlockRegex
from model.tools.search import Search
import model.tools.text_processor as tp
from model.tools.sentiment import sentiment_analysis, create_graphs
from model.tools import ngrams as ngr
from model.tools import criteria

if TYPE_CHECKING:
    from matplotlib.figure import Figure

class Model():
    """
    This class handles the variables and methods
    composing the logic of the program.
    """
    def __init__(self):
        # settings
        self._settings = Settings()
        # loaded parameters
        self._patterns = {}
        self._criteria_ranking = []
        self._tags = {}
        # meta
        self._filename = ""
        #text processing
        self._text = ''
        self._text_blocks = []
        self._sorted_blocks = []
        self._sentences = []
        self._blob = None
        # counters
        self._tag_counters = {}
        self._category_counters = {}
        # search
        self._search_results = []
        self._searcher = Search(
            self._text_blocks,
            self._sorted_blocks,
            self._sentences,
            self._tags,
            self._settings
        )
        # results
        self._criteria_results = []
        self._ngrams_results = []
        self._sent_vader = []
        self._sent_blob = []
        self._fig = None
        # flags
        self._analized = False  # The text has been analyzed
        self._critical = False   # A critical setting has changed
        # nltk's data directory
        nltk.data.path.append(DATA_DIR)
        self._load_config()

    #*******************
    #*  Configuration  *
    #*******************

    def get_settings(self):
        # type: () -> dict[str, Any]
        """Get the current settings."""
        return deepcopy(self._settings.to_dict())

    def set_settings(self, **settings):
        """
        Change settings.

        Args:
            `tags` (bool): Enable tag matching.

            `decontract` (bool): Enable decontractions.

            `promising_contr` (bool): If uncertain, use the best guess
            for a contraction.

            `clean_words` (bool): Remove symbols from the text.

            `ngrams` (bool): Enable N-grams tab.

            `sentiment` (bool): Analyze sentiment.

            `font_size` (int): Base font size.

            `textfont` (int): Textbox font size.

            `font_family` (str): Font used on the program.
            "*Default*" (used as default) is tkinter's base font.

            `geometry` (str): Dimensions of the window.

            `win_state` (int): 1 -> maximized; 2 -> fullscreen; other -> windowed.

            `save_tags` (bool): Include tags on the results' report.

            `unmatched` (bool): Include unmatched patterns on the results' report.

            `graphs_on_excel` (bool): Include sentiment graphs on the results' report.

            `decoded_tags` (bool): Include meaning of tags on the results' report.

            `first_color` (str): First color of the gradient.

            `last_color` (str): Last color of the gradient.
        """
        try:
            if self._settings.changes_critical_setting(**settings):
                self._critical = True
            self._settings.set_setting(**settings)
        except AttributeError as exc:
            print(exc)

    def save_settings(self):
        # type: () -> dict[str, Any]
        """
        Save current settings.
        Returns the settings as a dictionary.
        """
        fo.save_json(SETTINGS_FILE, self._settings.to_dict())
        return self.get_settings()

    def _load_config(self):
        config_dict = fo.load_json(PATTERNS_FILE)
        self._patterns.update(config_dict["patterns"])
        self._criteria_ranking = config_dict["criteria ranking"]
        self._tags.update(config_dict["tags"])
        try:
            settings = fo.load_json(SETTINGS_FILE)
        except FileNotFoundError:
            settings = {}
        self.set_settings(**settings)

    #*********************
    #*  Text Processing  *
    #*********************

    def load_text(self, filepath):
        # type: (str) -> str
        """
        Read text from a file based on its extension.
        Supported extensions: `txt`, `doc/docx`, `pdf` and `rtf`.

        Args:
            `filename`: Name of the text file.

        Raises:
            `AttributeError`: File not compatible.

        Returns:
            `str`: Text inside the file.
        """
        text = fo.read_doc_text(filepath)
        self._filename = os.path.basename(filepath)
        return text

    def get_last_file(self):
        # type: () -> str
        """
        Return the name of the last text file loaded.
        Defaults to an empty string.
        """
        return self._filename

    def process_text(self, text):
        # type: (str) -> None
        """Process `text` for later analysis."""
        if self._is_same_text(text) and not self._critical:
            return

        text_blocks, blob, \
            sorted_blocks, tag_counters = tp.process_text(text, self._settings, self._tags.keys())

        self._text = tp.sanitize_text(text).lower()
        self._text_blocks[:] = text_blocks
        self._blob = blob
        self._sorted_blocks[:] = sorted_blocks
        self._tag_counters = tag_counters
        self._sentences[:] = blob.raw_sentences

        self._clear_results()
        self._analized = False
        self._critical = False

    def _is_same_text(self, text):
        return self._text == tp.sanitize_text(text).lower()

    #***************
    #*  Criterias  *
    #***************

    def analyze_criteria(self, text):
        # type: (str) -> dict[str,dict]
        """
        Get manipulation rates from `text`.
        """
        self.search_patterns(text)
        return self.get_criteria_results()

    def search_patterns(self, text):
        """
        Search manipulation patterns.
        """
        if self._critical or not self._is_same_text(text):
            self.process_text(text)

        if self._analized:
            return

        category_positions = {}
        search_results = []
        category_counters = {}

        for category, body in self._patterns.items():
            count_split = body["count_split"]
            category_positions[category] = set()

            for find_word in body["words"]:
                tags = find_word["tags"] if "tags" in find_word else []

                found_word = self._searcher.find_pattern(find_word["string"], tags)
                found_word.category = category
                if isinstance(found_word, StringBlockRegex) and "meaning"in body:
                    found_word.meaning = body["meaning"]

                search_results.append(found_word)

                self._update_category_positions(
                    found_word, category_positions[category], count_split
                )

            category_counters[category] = len(category_positions[category])

        self._search_results = search_results
        self._category_counters = category_counters
        self._analized = True

    def get_criteria_results(self):
        # type: () -> dict[str,dict]
        """
        Get criteria results.
        Returns `None` if patterns have not been searched.
        """
        if not self._search_results:
            return None

        self._criteria_results = criteria.get_criterias(
                categories_info=self._patterns,
                tag_counters=self._tag_counters,
                category_counters=self._category_counters,
                ranking=self._criteria_ranking
            )
        return deepcopy(self._criteria_results)

    def _update_category_positions(self, found_word, positions, count_split):
        # type: (StringBlock, set, bool) -> None
        if isinstance(found_word, StringBlockRegex):
            new_positions = itertools.chain.from_iterable(found_word.pos_text)
        else:
            new_positions = found_word.pos_text

        length = len(found_word.words)

        for first_pos in new_positions:
            all_pos = tuple(range(first_pos, first_pos + length))

            self._add_category_positions(all_pos, positions, count_split)

    def _add_category_positions(self, new_in, positions, count_split):
        # type: (tuple, set, bool) -> None

        if count_split: #category_positions is a set of numbers
            positions.update(list(new_in))
            return

        # positions contains tuples; they can overlap but never contain each other
        to_remove = set()
        for inside in positions:
            if set(inside).issubset(new_in): #* *inside* is more specific
                return

            if set(new_in).issubset(inside):
                # works in our case because a pattern never overlaps two of 'positions'
                to_remove.update([inside])

        positions.update([new_in])
        positions = positions.difference(to_remove)

    #*************
    #*  N-grams  *
    #*************

    def query_ngrams(self, n=2,
            min_length=None, max_length=None, frequency=None,
            stopwords=False, remove_words=None, contains_words=None
        ):
        # type: (int,int,int,int,bool,list[str],list[str]) -> list[tuple[tuple[str,str], int]]
        """
        Get n-grams from the current text

        Args:
            `n` (optional): Degree of the n-gram. Must be 2, 3 or 4.
            Defaults to 2.

            `min_length` (optional): Minimum length of the words on the ngrams.
            Defaults to None.

            `max_length` (optional): Maximum length of the words on the ngrams.
            Defaults to None.

            `frequency` (optional): Minimum frequency.
            Defaults to None.

            `stopwords` (optional): Apply a filter of stopwords from the Stopwords Corpus.
            Defaults to False.

            `remove_words` (optional): Words to remove.
            Defaults to None.

            `contains_words` (optional): Words to filter.
            Defaults to None.

        Raises:
            `AttributeError`: `remove_words` and `contains_words` cannot share words.

        Returns:
            `list[tuple[tuple[str,str], int]]`: List of ngrams
            with its words and frequency.
        """
        if len(set(remove_words).intersection(contains_words)) > 0:
            raise AttributeError("Cannot exclude and filter the same word.")

        words = [block.words[0] for block in self._text_blocks]
        finder = ngr.get_finder(words, n)

        if min_length or max_length:
            ngr.ngram_length(finder, min_length, max_length)
        if frequency and frequency > 0:
            ngr.ngram_frequency(finder, frequency)
        if stopwords:
            ngr.ngram_stopwords(finder)
        if contains_words:
            ngr.ngram_match_words(finder, contains_words)
        if remove_words:
            ngr.ngram_stopwords(finder, remove_words)

        return ngr.ngrams_from_finder(finder)

    def get_ngram_sentences(self, ngram):
        # type: (tuple[str]) -> list[str]
        """
        Return sentence/s where the ngram appears.

        Args:
            `ngram`: n-gram to find.

        Returns:
            `list[str]`: List of sentences. If an occurrence
            is "bridging" more than one, they will be concatenated.
            If no occurrence is found, returns an empty list.

        """
        return self._searcher.find_ngram_sentences(ngram)

    def add_ngrams_search(self, query, result):
        # type: (dict[str, Any], tuple[tuple[str,...], list[str]]) -> None
        """
        Store a query and its result.

        Args:
            `query`: Query on dictionary format.

            `result`: tuple((n-gram), [sentences])
        """
        self._ngrams_results.append((query, result))

    def remove_ngrams_result(self, index):
        # type: (int|list[int]) -> None
        """Remove a result or a list of results by index."""
        if isinstance(index, list):
            for i in index:
                self._ngrams_results.pop(i)
        else:
            self._ngrams_results.pop(index)

    #***************
    #*  Sentiment  *
    #***************

    def get_sentiment(self):
        #type: () -> tuple[list[float], list[tuple[float, float]]]
        """
        Do a sentiment analysis to the processed text
        with Vader and TextBlob.

        Returns:
            `list(float)`: Sentiment compound for each sentence
            analyzed with VADER.

            `list(tuple(float, float))`: List of tuples for each sentence
            analyzed with TextBlob.
        """
        if not self._sent_vader and not self._sent_blob:
            self._sent_vader, self._sent_blob = sentiment_analysis(self._blob)

        return deepcopy(self._sent_vader), deepcopy(self._sent_blob)

    def get_sentiment_graphs(self, copy=True):
        # type: (bool) -> Figure
        """Create a Matplotlib figure with sentiment results
        from VADER and TextBlob of the processed text.

        Args:
            `copy`: If False, returns the referenced figure.
            Defaults to True.

        Returns:
            `Figure`: Matplotlib figure.
        """
        if self._fig is None:
            self._fig = create_graphs(self._sent_vader, self._sent_blob)
        return deepcopy(self._fig) if copy else self._fig

    #*************
    #*  Results  *
    #*************

    def _clear_results(self):
        self._search_results.clear()
        self._criteria_results.clear()
        self._ngrams_results.clear()

        self._sent_vader.clear()
        self._sent_blob.clear()
        self._fig = None

    def save_results(self, filename):
        # type: (str) -> None
        """
        Save the results on an Excel document.

        Args:
            `filename`: path of the Excel document.

        Raises:
            `Exception`: An error has occurred while saving.
        """
        graphs_filename = ""
        if (self._settings.sentiment and self._settings.graphs_on_excel):
            graphs_file = tempfile.NamedTemporaryFile(suffix='.jpg', delete=False)
            self._fig.savefig(graphs_file)
            graphs_filename = graphs_file.name
            print(graphs_filename)

        try:
            fo.save_results(
                filename, deepcopy(self._patterns), deepcopy(self._tags), deepcopy(self._blob),
                deepcopy(self._settings), deepcopy(self._search_results),
                deepcopy(self._criteria_results),
                deepcopy(self._ngrams_results),
                graphs_filename=graphs_filename
            )
        except Exception as exc:
            raise Exception(f"Error saving results:\n{type(exc).__name__}: {exc}") from exc
        finally:
            if graphs_filename:
                graphs_file.close()
                os.remove(graphs_filename)


from django.contrib import admin
from django.urls import path, re_path
from api import views

urlpatterns = [
    path('admin/', admin.site.urls),

    #path('api/teachers/', TeacherList.as_view(), name='teacher-list'),
    #path('api/teachers/<int:pk>/', TeacherDetail.as_view(), name='teacher-detail'),
    path('api/students', views.students_list, name='student-list'),

     path('api/students/<int:pk>', views.student_get, name='student-detail'),
     path('api/studentUpdate/<int:pk>', views.student_update, name='studentUpdate'),
     path('api/studentDelete/<int:pk>/', views.student_delete, name='studentDelete'),

    #teacher's url
    path('api/teachers', views.teachers_list, name='teacher-list'),
    path('api/teachers/<int:pk>', views.teacher_get, name='teacher-detail'),
    path('api/teacherUpdate/<int:pk>', views.teacher_update, name='teacherUpdate'),
     path('api/teacherDelete/<int:pk>/', views.teacher_delete, name='teacherDelete'),
]


import cProfile, pstats, io, os

def profileit(func):
    def wrapper(*args, **kwargs):
        if profileit.enable_profile:
            datafn = f"{func.__name__}.profile{os.getpid()}"
            prof = cProfile.Profile()
            retval = prof.runcall(func, *args, **kwargs)
            s = io.StringIO()
            sortby = 'cumulative'
            ps = pstats.Stats(prof, stream=s).sort_stats(sortby)
            ps.print_stats()
            with open(datafn, 'w') as perf_file:
                perf_file.write(s.getvalue())
            return retval
        else:
            return func(*args, **kwargs)

    return wrapper

profileit.enable_profile = False


"""/app/__init__.py

Description: App factory for pictureGen service
Project: Fauxstrology
Author: Connor Johnstone
Date: 12/7/2019

"""
#=== Start imports ===#

# std lib
import logging
import os

# third party
from flask import Flask
import redis
import boto3

# local
from .routes import constellation

#=== End imports ===#

# Define Routes
ROUTE_TABLE = {
    '/': constellation,
}

# Set up configuration data
class Config:
    REDIS_DB = redis.Redis(host='redis', port=6379)
    session = boto3.Session(aws_access_key_id=os.getenv('AWS_ACCESS_KEY_ID'),
                            aws_secret_access_key=os.getenv('AWS_SECRET_ACCESS_KEY'),
    )
    S3 = session.client('s3')

def new_app():
    """ Creates a new flask app instance
    """
    app = Flask(__name__)

    # add config data
    app.config.from_object(Config())

    # Construct Routes
    for url in ROUTE_TABLE:
        app.add_url_rule(url, view_func=ROUTE_TABLE[url])

    return app


import sys
import itertools

sys.path.append("AdventOfCode/lib")
import adventutil

fetcher = adventutil.InputFetcher('2015', '13')
input = fetcher.fetch(rstrip=True, commasplit=False)

map={}
guests = set()
for line in input:
    (person1, _, direction, magnitude, _, _, _, _, _, _, person2) = line.split()
    person2 = person2[:-1]
    guests.add(person1)
    direction = 1 if direction == "gain" else -1
    map[f"{person1}:{person2}"] = direction * int(magnitude)
    map[f"me:{person1}"] = 0 # comment out for part 1
    map[f"{person1}:me"] = 0 # comment out for part 1
guests.add("me") # comment out for part 1

def score(person1, person2):
    return map[f"{person1}:{person2}"]

max_happiness = -999
for p in itertools.permutations(guests):
    happiness = 0
    for i in range(len(guests)-1):
        happiness += score(p[i], p[i+1])
        happiness += score(p[i+1], p[i])
    happiness += score(p[-1], p[0])
    happiness += score(p[0], p[-1])
    max_happiness = max([happiness, max_happiness])
print(max_happiness)


import RPi.GPIO as GPIO
import time
import requests
from helpers.file_utils import load_json_file, write_status_to_file

class MotionDetector:
    def __init__(self, config):
        self.config = config
        self.WEB_URL = f"http://{config['host']}:{config['port']}/{config['get']}"
        self.status_file_path = "config/status.json"
        self.last_detection_time = 0

        GPIO.setmode(GPIO.BOARD)
        GPIO.setup(config['SENSOR_PIN'], GPIO.IN)

    def printTime(self):
        current_time = time.strftime("%Y-%m-%d %H:%M:%S")
        print("Movement detected at", current_time)

    def sendHTTPRequest(self, url):
        try:
            response = requests.get(url)
            if response.status_code == 200:
                print("HTTP request sent successfully")
        except requests.exceptions.RequestException as e:
            print("Error sending HTTP request:", e)

    def motionDetected(self, iSensorPin):
        self.printTime()
        self.last_detection_time = time.time()
        self.sendHTTPRequest(self.WEB_URL)
        write_status_to_file(self.status_file_path, True, "Room is Open")
        time.sleep(self.config['sleep_interval'])

    def setupMotionDetection(self):
        try:
            GPIO.add_event_detect(self.config['SENSOR_PIN'], GPIO.RISING, callback=self.motionDetected)
            while True:
                current_time = time.time()
                if current_time - self.last_detection_time > self.config['detection_interval']:
                    write_status_to_file(self.status_file_path, False, "Room is Closed")
                time.sleep(1)
        except KeyboardInterrupt:
            print("Quit")
            GPIO.cleanup()

if __name__ == "__main__":
    config = load_json_file("config/config.json")
    detector = MotionDetector(config)
    detector.setupMotionDetection()


# -*- coding: utf-8 -*-

import colorsys
import numpy as np
import scipy.linalg
import rviz_client

class ForceDistributionViewer:
    """
        Singleton pattern
        Duplicated instantiation causes the error of ROS node intialization
    """

    _unique_instance = None

    def __new__(cls):
        raise NotImplementationError('Cannot generate instance by constructor')

    @classmethod
    def __internal_new__(cls):
        return super().__new__(cls)

    @classmethod
    def get_instance(cls):
        if not cls._unique_instance:
            cls._unique_instance = cls.__internal_new__()
            cls.rviz_client = rviz_client.RVizClient()
        return cls._unique_instance

    def publish_bin_state(self,
                          bin_state,
                          fmap,
                          draw_fmap=True,
                          draw_force_gradient=False,
                          draw_range=[0.5, 0.9]):
        self.rviz_client.delete_all()
        self.draw_bin(fmap)

        positions = fmap.get_positions()
        fvals = fmap.get_values()

        if bin_state is not None:
            self.draw_objects(bin_state, fmap)
        if draw_fmap:
            self.draw_force_distribution(positions, fvals, draw_range=draw_range)
        if draw_force_gradient:
            self.draw_force_gradient(positions, fvals)
        self.rviz_client.show()

    def draw_bin(self, fmap):
        scene = fmap.get_scene()
        if scene == 'seria_basket':
            mesh_file = 'seria_basket.dae'
            mesh_pose = ([0, 0, 0.73], [0, 0, 0.70711, 0.70711])
            scale = [1, 1, 1]
        elif scene == 'konbini_shelf':
            mesh_file = 'simple_shelf.obj'
            mesh_pose = ([0, 0, 0], [0, 0, 0, 1])
            scale = [0.01, 0.01, 0.01]
        else:
            print(f'unknown scene: {scene}')
            return

        self.rviz_client.draw_mesh(f"package://force_estimation/meshes_extra/{mesh_file}",
                                   mesh_pose,
                                   rgba=(0.5, 0.5, 0.5, 0.2),
                                   scale=scale)

    def draw_objects(self, bin_state, fmap):
        for object_state in bin_state:
            name, pose = object_state
            scene = fmap.get_scene()
            if scene == 'seria_basket':
                self.rviz_client.draw_mesh("package://force_estimation/meshes/ycb/{}/google_16k/textured.dae".format(name),
                                        pose,
                                        (0.5, 0.5, 0.5, 0.3))
            elif scene == 'konbini_shelf':
                self.rviz_client.draw_mesh("package://force_estimation/meshes/konbini/{}.obj".format(name),
                                        pose,
                                        (0.5, 0.5, 0.5, 0.3))

    def draw_force_distribution(self, positions, fvals, draw_range=[0.5, 0.9]):
        fvals = fvals.flatten()
        fmax = np.max(fvals)
        fmin = np.min(fvals)
        points = []
        rgbas = []
        if fmax - fmin < 1e-3:
            print('the range of force values too small')
            return

        for (x, y, z), f in zip(positions, fvals):
            if draw_range[0] <= f and f <= draw_range[1]:
                points.append([x, y, z])
                hue = max(0, (0.7 - f) / 0.7)
                r, g, b = colorsys.hsv_to_rgb(hue, 1, 1)
                rgbas.append([r, g, b, 1])

        self.rviz_client.draw_points(points, rgbas)

    def draw_vector_field(self, positions, values, scale=0.5):
        self.rviz_client.draw_arrows(positions, positions+values*scale)

    def draw_force_gradient(self, positions, fvals, scale=0.3, threshold=0.008):
        gxyz = np.gradient(- fvals)
        g_vecs = np.column_stack([g.flatten() for g in gxyz])
        pos_val_pairs = [(p, g) for (p, g) in zip(positions, g_vecs) if scipy.linalg.norm(g) > threshold]
        positions, values = zip(*pos_val_pairs)
        self.draw_vector_field(np.array(positions), np.array(values), scale=scale)


import time
import gevent

from Queue import Full
from gevent.lock import Semaphore
from gevent.event import Event
from sqlalchemy.pool import Pool

class GeventConnectionPool(Pool):
    def __init__(self, creator, recycle=-1, echo=None, use_threadlocal=False, logging_name=None, reset_on_return=True, listeners=None, events=None, _dispatch=None, _dialect=None, max_connections=None, reserve_connections=None, connection_idle_timeout=30, is_full_event='wait'):
        super(GeventConnectionPool, self).__init__(
            creator, recycle=recycle, echo=echo, use_threadlocal=use_threadlocal, logging_name=logging_name, reset_on_return=reset_on_return,
            listeners=listeners, events=events, _dispatch=_dispatch, _dialect=_dialect)

        self.max_connections = max_connections or 5
        self.reserve_connections = reserve_connections or 1
        self.connection_idle_timeout = connection_idle_timeout
        self.is_full_event = is_full_event

        self._lock = Semaphore()
        self._available = list()
        self._inuse = list()
        self._timeouter = gevent.spawn_later(self.connection_idle_timeout, self._timeout)
        self._is_not_full = Event()

    def recreate(self):
        self.logger.info("Pool recreating")
        return self.__class__(
            self._creator, recycle=self._recycle, echo=self.echo, use_threadlocal=self._use_threadlocal, logging_name=self._orig_logging_name, reset_on_return=self._reset_on_return, _dispatch=self.dispatch,
            _dialect=self._dialect, max_connections=self.max_connections, reserve_connections=self.reserve_connections, connection_idle_timeout=self.connection_idle_timeout, is_full_event=self.is_full_event)

    def dispose(self):
        with self._lock:
            while self._available:
                t, conn = self._available.pop()
                conn.close()
        self.logger.info("Pool disposed.")

    def _timeout(self):
        try:
            with self._lock:
                while self._available and len(self._available) + len(self._inuse) > self.reserve_connections and self._available[-1][0] + self.connection_idle_timeout < time.time():
                    t, connection = self._available.pop(-1)
                    connection.close()
                    self.logger.info('closing timed out connection after %0.2f seconds (%s available, %s in use)', time.time() - t, len(self._available), len(self._inuse))
        finally:
            self._timeouter = gevent.spawn_later(self.connection_idle_timeout, self._timeout)

    def _do_get(self):
        while True:
            try:
                with self._lock:
                    if self._available:
                        t, connection = self._available.pop(0)
                        self.logger.debug('pop connection (%s available, %s in use)', len(self._available), len(self._inuse))
                    else:
                        if self.max_connections is not None and len(self._inuse) >= self.max_connections:
                            self._is_not_full.clear()
                            raise Full('max connections of %s reached' % self.max_connections)
                        connection = self._create_connection()
                        self.logger.info('new connection (%s available, %s in use)', len(self._available), len(self._inuse) + 1)
                    self._inuse.append(connection)
            except Full:
                if self.is_full_event == 'wait':
                    self.logger.warning('pool full. waiting for available slot (%s available, %s in use)', len(self._available), len(self._inuse))
                    self._is_not_full.wait()
                else:
                    raise
            else:
                return connection

    def _do_return_conn(self, connection):
        with self._lock:
            self.logger.debug('release connection (%s available, %s in use)', len(self._available), len(self._inuse))
            self._inuse.remove(connection)
            self._available.insert(0, (time.time(), connection))
            self._is_not_full.set()


import spacy
import pandas as pd

# Load a blank spaCy model or an existing one
nlp = spacy.blank('en')

# Create a new EntityRecognizer in the spaCy model pipeline
entity_ruler = nlp.create_pipe('entity_ruler')

df = pd.read_csv("gazzette/NZ_geo_names.txt", sep="\t", header=None)

place_names = df.iloc[:, 1].tolist()   # assuming the column you want to extract is the second column (index 1)

patterns=[]
for rows in place_names:
    patterns.append({'label': 'LOC', 'pattern': rows})
print(len(patterns))

entity_ruler.add_patterns(patterns)

# Add the EntityRecognizer to the spaCy model pipeline
nlp.add_pipe(entity_ruler)

# # Test the model
doc = nlp('The Selwyn River flows through North Canterbury.')
for ent in doc.ents:
    print(ent.text, ent.label_)


from django.contrib.auth.decorators import login_required
from django.shortcuts import render
from .models import Note

def index(request):
    return render(request, "index.html")

def about(request):
    return render(request, "about.html")

@login_required(login_url="/login/")
def home(request):
    # Get the latest notes
    latest_notes = Note.objects.filter(public=True).order_by("-pub_date")[:30]

    context = {
        "notes": latest_notes
    }

    return render(request, "home.html", context=context)

@login_required(login_url="/login/")
def add_note(request):
    current_user = request.user
    if "note" in request.POST and "title" in request.POST:
        public = "public" in request.POST
        text = request.POST["note"]
        title = request.POST["title"]
        if len(text) > 0 and len(title):
            Note(author=current_user, text=text, title=title, public=public).save()
            return render(request, "added_note.html")

    # On error show the form again
    return render(request, "add_note.html")

@login_required(login_url="/login/")
def personal_notes(request):
    # Get all personal notes
    user_personal_notes = Note.objects.filter(author=request.user).order_by("-pub_date")

    context = {
        "notes": user_personal_notes
    }

    return render(request, "home.html", context=context)


from rest_framework.relations import PrimaryKeyRelatedField
from rest_framework.serializers import ModelSerializer

from main.models.organization import Organization
from main.models.user import User

class AdminOrganizationSerializer(ModelSerializer):
    class Meta:
        fields = ("code", "id", "name", "user")
        model = Organization

    user = PrimaryKeyRelatedField(queryset=User.objects.all(), write_only=True)

    def create(self, validated_data):
        user = validated_data.pop("user")
        organization = super().create(validated_data)
        organization.staff_set.create(
            does_organization_agree=True, does_user_agree=True, user=user
        )
        return organization


import os
import requests
import json
import zipfile

class AlexaRequest:
    BASEPATH = os.path.dirname(os.path.realpath(__file__))

    def __init__(self, url):
        self.url = url

    def get_images(self, imgname):
        query = { 'img': imgname }
        r = requests.get(self.url + "image", params=query)
        print(r)

        if r.status_code == 404:
            post_image(self.BASEPATH + imgname)
        elif r.status_code == 200:
            r.raw.decode_content = True
            with open(imgname, 'wb') as f:
                f.write(r.content)
            with zipfile.ZipFile(imgname, 'r') as zip_ref:
                zip_ref.extractall(self.BASEPATH)
        else:
            raise RuntimeError()

    def post_image(self, imagename, tag):
        data = { 'tag': tag }
        files = { 'image' : open(self.BASEPATH + '/' + imagename, 'rb') }

        r = requests.post(self.url + "image", files=files, data=data)
        print(r)

    def get_strings(self, key):
        query = { 'key': key }
        r = requests.get(self.url + "strings", params=query)
        try:
            json = r.json()
            return json["strings"]
        except:
            if r.status_code == 404:
                print("key not found")
            else:
                print("server error")

    def del_strings(self, key):
        query = { 'key' : key }
        r = requests.delete(self.url + "strings", params=query)

    def post_string(self, key, string):
        data = { "key" : key, "string" : string }
        requests.post(self.url + "strings", data)

def string_test(ar):
    # Send some example strings to server
    ar.post_string("sentences", "this is an example sentence")
    ar.post_string("sentences", "this is another example sentence")

    # Retreive and print strings
    strings = ar.get_strings("sentences")
    print(strings)

    # Delete the entries
    ar.del_strings("sentences")
    strings = ar.get_strings("sentences")
    print(strings)

def image_test(ar):
    ar.post_image("graph.png", "math")
    ar.get_images("math")

if (__name__ == "__main__"):
    print("running AlexaRequest example")

    # Example server url
    server_url = "http://10.0.0.194:5000/"

    # Create request instance
    ar = AlexaRequest(server_url)

    #string_test(ar)
    image_test(ar)


import requests
import re
import tkinter as tk

class RESTAdmin(tk.Frame):
    def __init__(self, master=None):
        tk.Frame.__init__(self, master)
        self.grid()
        self.main_frame = tk.Frame(self)
        self.main_frame.grid()

        self.createVariables()
        self.createWidgets()

    def createVariables(self):
        self.sessiontoken_var = tk.StringVar()
        self.servicekey_var = tk.StringVar()
        self.profilename_var = tk.StringVar()

        self.send_messagevar = tk.StringVar()
        self.baseip_var = tk.StringVar()
        self.baseip_var.set("10.173.135.15")

    def createWidgets(self):
        self.sessiontoken_label = tk.Label(self.main_frame,text="PTLSSessionToken: ")
        self.sessiontoken_label.grid(sticky=tk.NW,row=0,column=0)
        self.servicekey_label = tk.Label(self.main_frame,text="ServiceKey: ")
        self.servicekey_label.grid(sticky=tk.NW,row=1,column=0)
        self.profilename_label = tk.Label(self.main_frame,text="MerchantProfileId: ")
        self.profilename_label.grid(sticky=tk.NW,row=2,column=0)
        self.baseip_label = tk.Label(self.main_frame,text="Host: ")
        self.baseip_label.grid(sticky=tk.NW,row=3,column=0)

        self.sessiontoken_entry = tk.Entry(self.main_frame,textvariable=self.sessiontoken_var,width=70)
        self.sessiontoken_entry.grid(sticky=tk.NW,row=0,column=1)
        self.servicekey_entry = tk.Entry(self.main_frame,textvariable=self.servicekey_var,width=70)
        self.servicekey_entry.grid(sticky=tk.NW,row=1,column=1)
        self.profilename_entry = tk.Entry(self.main_frame,textvariable=self.profilename_var,width=70)
        self.profilename_entry.grid(sticky=tk.NW,row=2,column=1)
        self.baseip_entry = tk.Entry(self.main_frame,textvariable=self.baseip_var,width=70)
        self.baseip_entry.grid(sticky=tk.NW,row=3,column=1)

        self.send_button = tk.Button(self.main_frame,text="Send Request",command=self.sendRequest)
        self.send_button.grid(row=4,column=0,columnspan=2)

        self.send_message = tk.Message(self.main_frame,textvariable=self.send_messagevar,aspect=1000)
        self.send_message.grid(row=5,column=0,columnspan=2)

        self.result_canvas = tk.Canvas(self.main_frame,relief=tk.RIDGE,bd=2,width=600)

    def sendRequest(self):
        for Id in self.result_canvas.find_all():
            self.result_canvas.delete(Id)

        if self.sessiontoken_var.get() == "" or self.servicekey_var.get() == "":
            self.send_messagevar.set("PTLSSessionToken and ServiceKey are required!")
            self.send_message["fg"] = "red"
            return
        url = "http://" + self.baseip_var.get() + "/Admin/batch/resend?ptlsSessionToken=" + self.sessiontoken_var.get() + "&serviceKey=" + self.servicekey_var.get()
        if self.profilename_entry.get() != "":
            url += "&profileName=" + self.profilename_entry.get()

        try:
            r = requests.post(url)
            successcode = re.search('(?:<IsSuccess>)(\w+)',r.text)
            respmessage = re.search('(?:<Message>)([\w ]+)',r.text)
            self.send_messagevar.set("ReSend Batch Request Sent.")
            self.send_message["fg"] = "blue"
            self.result_canvas.create_text(5,5,anchor=tk.NW,text="HTTP Status Code = " + str(r.status_code) + "\nIsSuccess = " + successcode.group(1) + "\nMessage = " +respmessage.group(1))
        except requests.exceptions.RequestException as errorstr:
            self.result_canvas.create_text(5,5,anchor=tk.NW,text=errorstr,width=600)
            self.send_messagevar.set("An Exception Occurred.")
            self.send_message["fg"] = "red"
        self.result_canvas.grid(sticky=tk.N,row=6,column=0,columnspan=2)

restgui = RESTAdmin()
restgui.master.title("TPS:Re-Send Batch")
restgui.mainloop()


import pylab as pl
import numpy as np
from astropy import units as u
from astropy.table import Table, Column

from cfe_global_plots import surfg_arr, fbound as global_fbound
from cfe_local_plots import fbound as local_fbound

from matplotlib.colors import LinearSegmentedColormap

from mpl_plot_templates import adaptive_param_plot

pl.matplotlib.rcParams['xtick.direction'] = 'in'
pl.matplotlib.rcParams['ytick.direction'] = 'in'
pl.rc('font', family='serif')
pl.matplotlib.rcParams['font.serif'] = 'Times New Roman'
pl.matplotlib.rcParams['mathtext.fontset'] = 'dejavuserif'
pl.matplotlib.rcParams['mathtext.rm'] = 'dejavuserif'
pl.rc('text', usetex=True)

# Adamo+ 2015 for most of the data
# Kruijssen & Bastian 2016 for gas surface densities
galaxy_data = {'SMC': {'SigSFR':0.001, 'Gamma':4.2, 'eGamma':(0.3,0.2), 'SigGas': 10**0.96, 'distance': 60*u.kpc},
               'LMC': {'SigSFR':1.52e-3, 'Gamma':5.8, 'eGamma':(0.5,0.5), 'SigGas': 10**1.04, 'distance': 50*u.kpc},
               'NGC 3256': {'SigSFR': 0.62, 'Gamma': 22.9, 'eGamma':(9.8, 7.3), 'SigGas': 10**2.1, 'distance': 19.8*u.Mpc},
               'Solar Neighborhood': {'SigSFR': 0.012, 'Gamma': 7.0, 'eGamma': (3,7), 'SigGas': 10**0.97, 'distance':500*u.pc},
               'NGC 45': {'SigSFR': 1.02e-3, 'Gamma': 5.2, 'eGamma':(0.3,0.3), 'SigGas': 10**0.67, 'distance': 6.8*u.Mpc},
               'NGC 1313': {'SigSFR':0.011, 'Gamma':3.2, 'eGamma':(0.2,0.2), 'SigGas': 10**0.88, 'distance': 4.15*u.Mpc},
               'NGC 4395': {'SigSFR':4.66e-3, 'Gamma':1, 'eGamma':(0.6,0.6), 'SigGas': 10**0.59, 'distance': 4.23*u.Mpc},
               'NGC 7793': {'SigSFR':6.51e-3, 'Gamma':2.5, 'eGamma':(0.3,0.3), 'SigGas': 10**0.84, 'distance': 3.775*u.Mpc},
               'NGC 4449': {'SigSFR':0.04, 'Gamma':9.0, 'eGamma':(0,0), 'SigGas': np.nan, 'distance': 3.98*u.Mpc},
               'NGC 1569': {'SigSFR':0.03, 'Gamma':13.9, 'eGamma':(0.8,0.8), 'SigGas': 10**1.33, 'distance': 2.63*u.Mpc},
               #'Dwarf Sample': {'SigSFR':, 'Gamma':, 'eGamma':, 'SigGas': 0},
               'IC 10': {'SigSFR':0.03, 'Gamma':4.2, 'eGamma':(0,0), 'SigGas': np.nan, 'distance': 0.792*u.Mpc},
               'ESO 338': {'SigSFR': 1.55, 'Gamma':50, 'eGamma':(10,10), 'SigGas': np.nan, 'distance': np.nan*u.Mpc},
               'Haro 11': {'SigSFR': 2.16, 'Gamma':50, 'eGamma':(15,13), 'SigGas': 10**1.98, 'distance': 92*u.Mpc}, # wikipedia
               'ESO 185-IG13': {'SigSFR': 0.52, 'Gamma':26, 'eGamma':(5,5), 'SigGas': np.nan, 'distance': 80*u.Mpc}, # Adamo+ 2011
               'MRK 930': {'SigSFR':0.59, 'Gamma':25, 'eGamma':(10,10), 'SigGas': np.nan, 'distance': 72*u.Mpc}, # Adamo+ 2011
               'SBS 0335-052E': {'SigSFR':0.95, 'Gamma':49, 'eGamma':(15,15), 'SigGas': np.nan, 'distance': 54.3*u.Mpc}, # Pustilnik+ 2000
               'NGC 2997': {'SigSFR':9.4e-3, 'Gamma':10, 'eGamma':(2.6,2.6), 'SigGas': np.nan, 'distance': 12.2*u.Mpc}, # Hess+ 2009
               'M83 center': {'SigSFR':0.54, 'Gamma':26.7, 'eGamma':(4,5.3), 'SigGas': np.nan, 'distance': 4.85*u.Mpc},
               'M83 middle': {'SigSFR':0.013, 'Gamma':18.2, 'eGamma':(3,3), 'SigGas': np.nan, 'distance': 4.85*u.Mpc},
               'M83 outer': {'SigSFR':0.013, 'Gamma':5.6, 'eGamma':(0.6,0.6), 'SigGas': np.nan, 'distance': 4.85*u.Mpc},
               'NGC 6946': {'SigSFR':4.6e-3, 'Gamma':12.5, 'eGamma':(2.5,1.8), 'SigGas': 10**1.30, 'distance': 5.9*u.Mpc}, # Karachentsev+ 2000
              }

m83surf = Table.read('m83_profiles.csv')
m83cfe = Table.read('m83_profiles_adamo.txt', format='ascii')

surfs = []
for row in m83cfe:
    sel = (m83surf['Radius'] < row['outer']) & (m83surf['Radius'] > row['inner'])
    avsurf = m83surf[sel]['Surfdens_H2'].mean()
    surfs.append(avsurf)

m83cfe.add_column(Column(name='surfdens_h2', data=surfs))

siggas = np.array([x['SigGas'] for x in galaxy_data.values()])*u.M_sun/u.pc**2
sigsfr = np.array([x['SigSFR'] for x in galaxy_data.values()])*u.M_sun/u.yr
gamma = np.array([x['Gamma'] for x in galaxy_data.values()])
egamma_low,egamma_high = np.array([x['eGamma'] for x in galaxy_data.values()]).T
distance = u.Quantity([x['distance'] for x in galaxy_data.values()])

sgrb2_data = Table.read('../tables/cluster_mass_estimates_cfe.csv')
cfe_sb2 = sgrb2_data[r'$M_{\rm inferred}$'][-1] / sgrb2_data[r'$M_{\rm inferred}$'][-2] * 100
cfe_sb2_low = 37
cfe_sb2_high = 43
cfe_sb2_yerr = np.array([[cfe_sb2-cfe_sb2_low, cfe_sb2_high-cfe_sb2]]).T
cfe_sb2 = 37
cfe_sb2_yerr = 7 # updated based on arguments in text...

# the average surface density (in dense gas) across the "stream", N(H2) ~ 5e22 cm^-2
sgrb2_surfdens = 1e3 * u.Msun/u.pc**2

# # this is not right...
# this number is an estimate of the immediate, local SFR as measured within Sgr
# B2 over a 15x15pc region from my data directly.  It's not right because the
# Kruijssen theory is based on the global conditions, not local
# sgrb2_sfr_surfdens = (0.062 * u.Msun/u.yr/(15*u.pc)**2).to(u.Msun/u.yr/u.kpc**2)
# esgrb2_sfr_surfdens = (0.1 * 0.062 * u.Msun/u.yr/(15*u.pc)**2).to(u.Msun/u.yr/u.kpc**2)

sgrb2_distance = (8.5*u.kpc).to(u.Mpc)

# this is the "correct" surface density to use averaged over the stream area
stream_area = (120**2-60**2)*np.pi*u.pc**2
sgrb2_sfr_surfdens = (0.1 * u.Msun/u.yr/(stream_area)).to(u.M_sun/u.yr/u.kpc**2)
min_stream_area, max_stream_area = (100**2-80**2)*np.pi*u.pc**2, (140**2-60**2)*np.pi*u.pc**2
esgrb2_sfr_surfdens = np.array(((sgrb2_sfr_surfdens - (0.1 * u.Msun/u.yr/(max_stream_area)).to(u.M_sun/u.yr/u.kpc**2)).value,
                               ((0.1 * u.Msun/u.yr/(min_stream_area)).to(u.M_sun/u.yr/u.kpc**2) - sgrb2_sfr_surfdens).value))
print("Sgr B2 SFR surface density = {0} + {1} - {2}".format(sgrb2_sfr_surfdens, esgrb2_sfr_surfdens[1], esgrb2_sfr_surfdens[0]))
print("Sgr B2  surface density = {0} */ {1}".format(sgrb2_surfdens, 1.65))

fig = pl.figure(1)
fig.clf()
ax = fig.gca()
ax.errorbar(sigsfr.value, gamma, yerr=np.array([egamma_low, egamma_high]),
            marker='s', linestyle='none', markeredgecolor='k', alpha=0.6)
ax.errorbar(sgrb2_sfr_surfdens.value, cfe_sb2, xerr=np.array([esgrb2_sfr_surfdens]).T,
            yerr=cfe_sb2_yerr,
            marker='o', linestyle='none', markeredgecolor='r',
            color='orange',
            markerfacecolor='orange')
ax.set_xscale('log')
ax.set_yscale('log')
ax.set_xlabel("$\Sigma_{\mathrm{SFR}}$ [M$_\odot$ yr$^{-1}$ pc$^{-2}$]", fontsize=18)
ax.set_ylabel("$\Gamma$ (percent)", fontsize=22)
ax.set_ylim(1,100)
ax.tick_params(labelsize=18)
fig.savefig('GammaVsSigmaSFR.pdf', bbox_inches='tight')

fig2 = pl.figure(2)
fig2.clf()
ax2 = fig2.gca()
ax2.errorbar(distance.to(u.Mpc).value, gamma, yerr=np.array([egamma_low, egamma_high]),
             marker='s', linestyle='none', markeredgecolor='k', alpha=0.6)
ax2.errorbar(sgrb2_distance.value, cfe_sb2,
             yerr=cfe_sb2_yerr,
             marker='o', linestyle='none', markeredgecolor='r',
             color='orange',
             markerfacecolor='orange')
ax2.set_xscale('log')
ax2.set_yscale('log')
ax2.set_xlabel("Distance [Mpc]", fontsize=22)
ax2.set_ylabel("$\Gamma$ (percent)", fontsize=22)
ax2.tick_params(labelsize=18)
ax2.set_ylim(1,100)
fig2.savefig('GammaVsDistance.pdf', bbox_inches='tight')

fig3 = pl.figure(3)
fig3.clf()
ax3 = fig3.gca()
ax3.errorbar(siggas.value, gamma, yerr=np.array([egamma_low, egamma_high]),
             marker='s', linestyle='none', markeredgecolor='k', alpha=0.6)
ax3.errorbar(m83cfe['surfdens_h2'][m83cfe['zone'] == 'eqarea'],
             m83cfe['cfe'][m83cfe['zone'] == 'eqarea'],
             yerr=m83cfe['ecfe'][m83cfe['zone'] == 'eqarea'],
             color='g',
            )
ax3.set_xscale('log')
ax3.set_yscale('log')
ax3.set_xlabel("$\Sigma_{\mathrm{gas}}$ [M$_\odot$ pc$^{-2}$]", fontsize=22)
ax3.set_ylabel("$\Gamma$ (percent)", fontsize=22)
ax3.tick_params(labelsize=18)
ax3.set_ylim(1,100)
ax3.set_xlim(0.5, 7e3)
fig3.savefig('GammaVsSigmaGas_extragalonly.pdf', bbox_inches='tight')
fig3.savefig('GammaVsSigmaGas_extragalonly.svg', bbox_inches='tight')

sb2pt = ax3.errorbar(sgrb2_surfdens.value, cfe_sb2,
                     xerr=np.array([[sgrb2_surfdens.value-sgrb2_surfdens.value/1.65,
                                     sgrb2_surfdens.value*1.65-sgrb2_surfdens.value]]).T,
                     yerr=cfe_sb2_yerr, marker='o', linestyle='none',
                     markeredgecolor='r', color='orange',
                     markerfacecolor='orange')
ax3.set_ylim(1,100)
ax3.set_xlim(0.5, 7e3)
fig3.savefig('GammaVsSigmaGas.pdf', bbox_inches='tight')
fig3.savefig('GammaVsSigmaGas.svg', bbox_inches='tight')

bins = 10**np.mgrid[1.5:3.5:0.25, 0.5:2:0.1]
bins = np.array([np.logspace(1.5,4,15), np.logspace(0.5,2,15)])

cdict1 = {'red':   ((0.0, 1.0, 1.0),
                    (1.0, 1.0, 1.0)),

          'green': ((0.0, 0.0, 0.0),
                    (1.0, 0.0, 0.0)),

          'blue':  ((0.0, 0.0, 0.0),
                    (1.0, 0.0, 0.0))
        }
cdictblue = {'red':   ((0.0, 0.0, 0.0),
                       (1.0, 0.0, 0.0)),

             'green': ((0.0, 0.0, 0.0),
                       (1.0, 0.0, 0.0)),

             'blue':  ((0.0, 1.0, 1.0),
                       (1.0, 1.0, 1.0))
        }
cm_red = LinearSegmentedColormap('red', cdict1)
cm_blue = LinearSegmentedColormap('blue', cdictblue)

#ax3.plot((surfg_arr*u.kg/u.m**2).to(u.M_sun/u.pc**2).value, cfes, 'k.', alpha=0.25, zorder=-5)
rslt = adaptive_param_plot((surfg_arr*u.kg/u.m**2).to(u.M_sun/u.pc**2).value,
                           global_fbound, marker='none',
                           #levels=[1-0.95,1-0.68])
                           #colors=['b']*15,
                           cmap=cm_red,
                           bins=bins,
                           percentilelevels=[0.05, 0.32],
                          )
rslt2 = adaptive_param_plot((surfg_arr*u.kg/u.m**2).to(u.M_sun/u.pc**2).value,
                            local_fbound, marker='none',
                            #levels=[1-0.95,1-0.68])
                            #colors=['r']*15,
                            cmap=cm_blue,
                            linestyles='dotted',
                            bins=bins,
                            percentilelevels=[0.05, 0.32],
                           )
ax3.set_ylim(1,100)
ax3.set_xlim(0.5, 7e3)

fig3.savefig('GammaVsSigmaGas_withTheory.pdf', bbox_inches='tight')
fig3.savefig('GammaVsSigmaGas_withTheory.svg', bbox_inches='tight')

for x in sb2pt.get_children():
    x.set_visible(False)

fig3.savefig('GammaVsSigmaGas_withTheory_withoutSB2.pdf', bbox_inches='tight')
fig3.savefig('GammaVsSigmaGas_withTheory_withoutSB2.svg', bbox_inches='tight')

fbound_median = np.nanmedian(global_fbound)
fbound_errmin = fbound_median - np.nanpercentile(global_fbound, 16.)
fbound_errmax = np.nanpercentile(global_fbound, 84.) - fbound_median

print('fbound_global: %.2f+%.2f-%.2f' % (np.around(fbound_median,2),np.around(fbound_errmax,2),np.around(fbound_errmin,2)))

fbound_median = np.nanmedian(local_fbound)
fbound_errmin = fbound_median - np.nanpercentile(local_fbound, 16.)
fbound_errmax = np.nanpercentile(local_fbound, 84.) - fbound_median

print('fbound_local: %.2f+%.2f-%.2f' % (np.around(fbound_median,2),np.around(fbound_errmax,2),np.around(fbound_errmin,2)))

fig4 = pl.figure(4, figsize=(10,4))
fig4.clf()
ax = fig4.add_subplot(1,3,2)
ax.errorbar(sigsfr.value, gamma, yerr=np.array([egamma_low, egamma_high]),
            marker='s', linestyle='none', markeredgecolor='k', alpha=0.6)
ax.errorbar(sgrb2_sfr_surfdens.value, cfe_sb2, xerr=np.array([esgrb2_sfr_surfdens]).T,
            yerr=cfe_sb2_yerr,
            marker='o', linestyle='none', markeredgecolor='r',
            color='orange',
            markerfacecolor='orange')
ax.set_xscale('log')
ax.set_yscale('log')
ax.set_xlabel("$\Sigma_{\mathrm{SFR}}$ [M$_\odot$ yr$^{-1}$ pc$^{-2}$]", fontsize=18)
ax.set_ylabel("$\Gamma$ (percent)", fontsize=18)
ax.set_ylim(1,100)
ax.set_xlim(6e-4,15)
ax.tick_params(labelsize=14)
[xx.set_visible(False) for xx in ax.get_yticklabels()]

ax2 = fig4.add_subplot(1,3,3)
ax2.errorbar(distance.to(u.Mpc).value, gamma, yerr=np.array([egamma_low, egamma_high]),
             marker='s', linestyle='none', markeredgecolor='k', alpha=0.6)
ax2.errorbar(sgrb2_distance.value, cfe_sb2,
             yerr=cfe_sb2_yerr,
             marker='o', linestyle='none', markeredgecolor='r',
             color='orange',
             markerfacecolor='orange')
ax2.set_xscale('log')
ax2.set_yscale('log')
ax2.set_xlabel("Distance [Mpc]", fontsize=18)
#ax2.set_ylabel("$\Gamma$", fontsize=18)
ax2.tick_params(labelsize=14)
ax2.set_ylim(1,100)
[xx.set_visible(False) for xx in ax2.get_yticklabels()]

ax3 = fig4.add_subplot(1,3,1)
ax3.errorbar(siggas.value, gamma, yerr=np.array([egamma_low, egamma_high]),
             marker='s', linestyle='none', markeredgecolor='k', alpha=0.6)
ax3.errorbar(sgrb2_surfdens.value, cfe_sb2,
             xerr=np.array([[sgrb2_surfdens.value-sgrb2_surfdens.value/1.65, sgrb2_surfdens.value*1.65-sgrb2_surfdens.value]]).T,
             yerr=cfe_sb2_yerr,
             marker='o', linestyle='none', markeredgecolor='r',
             color='orange',
             markerfacecolor='orange')
ax3.errorbar(m83cfe['surfdens_h2'][m83cfe['zone'] == 'eqarea'],
             m83cfe['cfe'][m83cfe['zone'] == 'eqarea'],
             yerr=m83cfe['ecfe'][m83cfe['zone'] == 'eqarea'],
             color='g',
            )
ax3.set_xscale('log')
ax3.set_yscale('log')
ax3.set_xlabel("$\Sigma_{\mathrm{gas}}$ [M$_\odot$ pc$^{-2}$]", fontsize=18)
ax3.tick_params(labelsize=14)
ax3.set_ylim(1,100)
ax3.set_xlim(0.5, 7e3)
#[xx.set_visible(False) for xx in ax3.get_yticklabels()]

rslt = adaptive_param_plot((surfg_arr*u.kg/u.m**2).to(u.M_sun/u.pc**2).value,
                           global_fbound, marker='none',
                           #levels=[1-0.95,1-0.68])
                           #colors=['b']*15,
                           cmap=cm_red,
                           bins=bins,
                           percentilelevels=[0.05, 0.32],
                          )
rslt2 = adaptive_param_plot((surfg_arr*u.kg/u.m**2).to(u.M_sun/u.pc**2).value,
                            local_fbound, marker='none',
                            #levels=[1-0.95,1-0.68])
                            #colors=['r']*15,
                            cmap=cm_blue,
                            linestyles='dotted',
                            bins=bins,
                            percentilelevels=[0.05, 0.32],
                           )
ax3.set_ylim(1,100)
ax3.set_xlim(0.5, 7e3)
ax3.set_ylabel("$\Gamma$ (percent)", fontsize=18)

fig4.subplots_adjust(wspace=0)

fig4.savefig('GammaVsEverything_3panel.pdf', bbox_inches='tight')
fig4.savefig('GammaVsEverything_3panel.svg', bbox_inches='tight')


from django.contrib import admin
from .models import Event, Profile

class EventAdmin(admin.ModelAdmin):
    list_display = ('title', 'slug', 'author', 'date_time', 'published')
    list_filter = ('published', 'date_time', 'author')
    search_fields = ('title', 'tags', 'body')
    prepopulated_fields = {'slug': ('title',)}
    date_hierarchy = 'date_time'
    ordering = ['published', 'date_time']

admin.site.register(Event, EventAdmin)
admin.site.register(Profile)
# Register your models here.


from re import template
import shutil
from flask import Flask, render_template, request, redirect, url_for, flash, send_from_directory
from flask_hcaptcha import hCaptcha
from dotenv import load_dotenv
import os
import uuid
import zipfile
import subprocess
load_dotenv('conf.env')

app = Flask(__name__)
app.config['HCAPTCHA_ENABLED'] = True
app.config['HCAPTCHA_SITE_KEY'] = os.getenv('HCAPTCHA_SITE_KEY')
app.config['HCAPTCHA_SECRET_KEY'] = os.getenv('HCAPTCHA_SECRET_KEY')
app.config['UPLOAD_FOLDER'] = '/uploads'
app.config['TEMPLATE_FOLDER'] = '/render_templates'
app.secret_key = os.getenv('FLASK_SECRET_KEY')
app.config['MAX_CONTENT_LENGTH'] = 10 * 1024 * 1024    # 10 Mb limit
hcaptcha = hCaptcha(app)

def get_template_list():
    return os.listdir(app.config['TEMPLATE_FOLDER'])

@app.route('/')
def index():
    return render_template('index.html', templates=get_template_list())

@app.route('/upload', methods=['POST'])
def upload_to_convert():
    """
    Accept a ZIP file upload.
    Check the captcha, then store the file in a temporary directory
    and redirect to the unzipping page.
    """
    if not hcaptcha.verify():
        flash('Invalid captcha')
        return redirect(url_for('index'))

    if 'file' not in request.files:
        flash('No file uploaded')
        return redirect(url_for('index'))

    template = request.form.get('template')
    if template not in get_template_list():
        flash('Invalid compilation template')
        return redirect(url_for('index'))

    # Save the file to disk
    file = request.files['file']
    name = str(uuid.uuid4())
    file.save(os.path.join(app.config['UPLOAD_FOLDER'], name + '.zip'))
    return redirect(url_for('unzip', name=name, template=template))

@app.route('/unzip/<name>/<template>')
def unzip(name, template):
    """
    Unzip the uploaded file.
    When unzipping is successful, redirect to the conversion page.
    """

    try:
        with zipfile.ZipFile(os.path.join(app.config['UPLOAD_FOLDER'], name + '.zip')) as zip_file:
            # Make folder for the unzipped files
            os.makedirs(os.path.join(app.config['UPLOAD_FOLDER'], name))
            # Extract the files
            zip_file.extractall(os.path.join(app.config['UPLOAD_FOLDER'], name))
    except FileNotFoundError:
        flash('No such file')
        return redirect(url_for('index'))
    except zipfile.BadZipFile:
        flash('Invalid ZIP file')
        return redirect(url_for('index'))

    return redirect(url_for('convert', name=name, template=template))

@app.route('/convert/<name>/<template>')
def convert(name, template):
    """
    Put the render template into the unzipped folder and begin the conversion.
    """

    directory = os.path.join(app.config['UPLOAD_FOLDER'], name)
    os.chdir(directory)

    # Remove the user's Makefile if it exists
    try:
        os.remove('Makefile')
    except FileNotFoundError:
        pass

    template = template.replace('/', '')

    # Copy files from the template folder to the user's folder
    # without overwriting the existing files
    template_root = os.path.join(app.config['TEMPLATE_FOLDER'], template)
    if not os.path.exists(template_root):
        flash('Invalid compilation template')
        return redirect(url_for('index'))

    for directory, subdirectories, files in os.walk(template_root):
        # For every file:
        for file in files:
            print(file)
            # Get the absolute path to the file
            file_path = os.path.join(directory, file)

            # Get path to this file in the template directory
            relative_path = os.path.relpath(file_path, template_root)

            # If the path doesn't exist in the user directory, copy it
            # (to the current directory, because we are in the user directory)
            if not os.path.exists(relative_path):
                # Make the directory if it doesn't exist
                if os.path.dirname(relative_path):
                    os.makedirs(os.path.dirname(relative_path), exist_ok=True)
                # then copy the file
                shutil.copy(file_path, relative_path)

    # Run the makefile, capturing the output
    handle = subprocess.Popen(['make'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
    try:
        handle.wait(timeout=60)
    except subprocess.TimeoutExpired:
        handle.kill()
        flash('Compilation timed out')
        return redirect(url_for('index'))

    stdout = handle.stdout.read().decode('utf-8')
    stderr = handle.stderr.read().decode('utf-8')
    exit_code = handle.returncode

    # Now that the compilation is done, zip up the folder
    os.chdir('..')
    shutil.make_archive(name+'-output', 'zip', name)

    # Delete the unzipped folder
    #shutil.rmtree(name)

    # Render the result page
    return render_template('convert_result.html', name=name, stdout=stdout, stderr=stderr, exit_code=exit_code)

@app.route('/download/<name>')
def download_output(name):
    return send_from_directory(app.config['UPLOAD_FOLDER'], name + '-output.zip')

if __name__ == '__main__':
    app.run('0.0.0.0', 5000)


import os
import sys

from grpc.tools import protoc

def gen_prometheus_proto_py(proto_file_dir, output_dir):
    # Function For fb-internal build tools - open source should use this file
    # as a script
    protoc.main(
        (
            '',
            '-I' + proto_file_dir,
            '--python_out=' + output_dir,
            '--grpc_python_out=' + output_dir,
            os.path.join(proto_file_dir, 'metrics.proto'),
        ),
    )

if __name__ == '__main__':
    # ./gen_prometheus_proto.py <magma root> <output_dir>
    magma_root, out_dir = sys.argv[1], sys.argv[2]
    file_dir = os.path.join(magma_root, 'proto_files/orc8r/protos/prometheus')
    gen_prometheus_proto_py(file_dir, out_dir)


from __future__ import print_function
import string
from PIL import Image
from PIL import ImageFont
from PIL import ImageDraw
import numpy as np

MAC_FONTS = "/System/Library/Fonts/"
LINUX_FONTS = "/usr/share/fonts/"

def char_to_pixels(text, font, w, h):
    """
    Based on https://stackoverflow.com/a/27753869/190597 (jsheperd)
    """
    image = Image.new('L', (w, h), 1)
    draw = ImageDraw.Draw(image)
    draw.text((0, 0), text, font=font)
    arr = np.asarray(image)
    arr = np.where(arr, 0, 1)
    arr = arr[(arr != 0).any(axis=1)]
    return arr

def letter(arr):
    result = np.where(arr, '#', ' ')
    return '\n'.join([''.join(row) for row in result])

def sentence(arrs, letter_spaces=3, space_size=10, w=7, h=6):
    sentence = []
    for x, arr in enumerate(arrs):
        result = np.where(arr, '#', ' ')
        if result.size == 0:
            for i in range(h):
                sentence[i] += ' '*space_size

        else:
            if len(result) < h:
                for _ in range(h - len(result)):
                    result = [" "*w] + result

            for i, row in enumerate(result):
                if x == 0:
                    sentence += [""]

                sentence[i] += ''.join(row)
                sentence[i] += ' '*letter_spaces

    return sentence

def textMatrix(text):

    font = ImageFont.truetype(MAC_FONTS + "/Avenir.ttc", 15)
    w, h = font.getsize("A")
    h *= 2

    A = char_to_pixels("A".upper(), font, w, h)

    arrs = []
    to_print = text
    to_print = to_print.strip().upper()
    for c in to_print:
        arrs += [char_to_pixels(c.upper(), font, w, h)]

    return "\n".join(sentence(arrs, 3, 10, *A.shape))


from aws_cdk import (
    Duration,
    Stack,
    aws_cloud9_alpha as cloud9,
    CfnOutput,
    aws_ec2 as ec2
)
from constructs import Construct

class Cloud9Stack(Stack):

    def __init__(self, scope: Construct, construct_id: str, vpc_name: str = "AcrVPC", **kwargs) -> None:
        super().__init__(scope, construct_id, **kwargs)

         # Get existing VPC
        vpc = ec2.Vpc.from_lookup(self, vpc_name, is_default=False)
        print(self.account)
        c9env = cloud9.Ec2Environment(self, "Cloud9Environment",
                                    vpc=vpc,
                                    description="Jerome's cloud9 env.",
                                    subnet_selection=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC),
                                    instance_type=ec2.InstanceType("t3.medium"),
                                    automatic_stop=Duration.hours(1),
                                    ec2_environment_name="Cloud9AcrDemo",
                                    owner=cloud9.Owner.account_root(self.account),
                                    image_id=cloud9.ImageId.AMAZON_LINUX_2)
        CfnOutput(self, "URL", value=c9env.ide_url)
        CfnOutput(self, "EC2_ARN", value=c9env.ec2_environment_arn)


import os
import os.path

from google.appengine.ext import webapp
from google.appengine.ext.webapp import template
from google.appengine.ext.webapp.util import run_wsgi_app

from py import rhizoglobals

class DocHandler(webapp.RequestHandler):
  def get(self):
    docfile = self.request.path.replace('/doc', '')
    if not docfile or docfile == '/':
      docfile = '/index.html'

    template_values = rhizoglobals.DefaultTemplate(self.request)
    template_values.update({
      'docfile': docfile,
    })
    path = os.path.join(os.path.dirname(__file__),
                        '../../templates/doc%s' % docfile)
    self.response.out.write(template.render(path, template_values))

application = webapp.WSGIApplication(
    [('/doc/.*', DocHandler),
     ('/doc', DocHandler),],
    debug=rhizoglobals.appenginedebug)

def main():
    run_wsgi_app(application)

if __name__ == '__main__':
    main()