from django.core.paginator import Paginator, EmptyPage
from django.shortcuts import render, get_object_or_404
from django.http import HttpResponse, HttpResponseNotFound
from .models import User, Account

def index(request):
	users = User.objects.all().order_by('id')
	limit = 20

	page = request.GET.get('page', 1)
	paginator = Paginator(users, limit)
	paginator.baseurl = '/shop/?page='	#

	try:
		page = paginator.page(page)					# Page
	except EmptyPage:						# Exception as e:
		page = paginator.page(paginator.num_pages)

	return render(request, 'index.html', {
		'list_users' : page.object_list,
		'paginator' : paginator,
		'page' : page,
		})

def user_page(request, pk):
	user = get_object_or_404(User, pk=pk)
	users_accounts = Account.objects.all().filter(user_account=user)

	query_name = request.GET.get('query')						# Выбираем из reques'a запрос (Имя пользователя)
	list_found_users = User.objects.filter(name=query_name)		# <QuerySet [<User: name>]>

	not_found = ''
	if query_name and not list_found_users:
		not_found = 'Такого пользователя не существует. Уточните поиск.'

	list_accounts = []

	for users in list_found_users:
		list_accounts.append(users.account_set.all())

#	print('\n\n')
#	print(list_accounts)
##	print(list_accounts[0][0], list_accounts[0][1])
##	print(list_accounts[1])
#	print('\n\n')

	return render(request, 'user_detail.html', {
		'user' : user,
		'users_accounts' : users_accounts,

		'list_found_users' : list_found_users,
		'not_found' : not_found
		})

def accounts_page(request):
	pk_list = request.GET.getlist("accounts")
	print('\n\n   pk_list = {}   \n\n'.format(pk_list))

	if not pk_list:
		return HttpResponseNotFound("Ни один счёт не выбран")

	accounts = []
	total_value = 0
	for pk in pk_list:
		account = get_object_or_404(Account, pk=pk)			# Выбираю первый отмеченный счёт
		accounts.append(account)							# Собирают list (список) счетов
		total_value = total_value + account.value 			# Считаю сумму на счетах

	user_id = Account.objects.get(pk=pk).user_account_id	# Получаю id user'a у которого этот аккаунт
	user = get_object_or_404(User, pk=user_id)				# На основе user_id получаю самого user'a

	return render(request, 'account_detail.html', {
		'user' : user,
		'accounts' : accounts,
		'total_value' : total_value
		})

def account_page(request, pk):
	account = get_object_or_404(Account, pk=pk)
	us_id = Account.objects.get(pk=pk).user_account_id
	user = get_object_or_404(User, pk=us_id)

	return render(request, 'account_detail.html', {
		'user' : user,
		'account' : account
		})

#----------------------------------

def search_page(request):
	not_found = ''
	query = request.GET.get('q')
	list_users = User.objects.all().order_by('name')
	list_users = User.objects.filter(name=query)

	if query and not list_users:
		not_found = 'Такого пользователя не существует. Уточните поиск.'

	return render(request, 'search_page.html', {
		'list_users' : list_users,
		'not_found' : not_found
		})


# Title:  Python Script Sample To Manage BPS VE Licenses.
# Actions:
#   1. Login to BPS box
#   2. Get a list of the configured license servers
#   3. Add a specific license server if is not already added
#   4. Check the active licenses on the specific server
#   5. Activate deactivate licenses
#   6. Check floating license feature statistics (check how many are available and taken)

#================

########################################
import sys
import os
# Add bps_restpy libpath *required if the library is not installed
libpath = os.path.abspath(__file__+"/../../..")
sys.path.insert(0, libpath)
import logging

from restPyWrapper import BPS, pp
from bpsVELicense import BPSVELicenseManagement
########################################

########################################
# Demo script global variables
########################################
# bps system info
bps_system  = '<BPS_BOX_IP/HOSTNAME>'
bpsuser     = 'bps user'
bpspass     = 'bps pass'

mainLicenseServer = '<License_Server_IP/HOSTNAME>'
activationCode = '<Activation_Code>'

########################################

########################################
# Login to BPS box
bps = BPS(bps_system, bpsuser, bpspass)
bps.login()

# create an instance of the BPSVELicenseManagement admin class
licenseMngr = BPSVELicenseManagement(bps)

# list license servers added
licenseServers = licenseMngr.getLicenseServers()
print ("The following license servers are added: " + str(licenseServers))

# add desired server if not already there
if not mainLicenseServer in [item['host'] for item in licenseServers if 'host' in item]:
    licenseMngr.addLicenseServer(mainLicenseServer)

# identify active server, if is not the one desired set it
activeServer = ''
for server in licenseServers:
    if 'isActive' in server and server['isActive']:
        activeServer = server

if not activeServer == mainLicenseServer:
    licenseMngr.setLicenseServerActive(mainLicenseServer)

# identify active server and store the details in  activeServer
for server in licenseServers:
    if 'isActive' in server and server['isActive']:
        activeServer = server

# extracting the id from the activeServer dictionary. We'll  use the id to identify teh server
licenseServerId = activeServer['id']
print("Server ID: %s"%licenseServerId)

# get active licenses on the active server
serverActiveLicenses = licenseMngr.retrieveLicenses(activeServer['id'])
print("The following licenses are active: " + str(serverActiveLicenses))

# is online activation possible?
status = licenseMngr.isServerOnline(activeServer['id'])
if not 'canConnectToIxiaBackend' in status or not status['canConnectToIxiaBackend']:
    print ("This server requires offline activation/deactivation process.\
         Please open license administration and follow the steps indicated.")
    print ("The rest of the steps are only aplicable to online servers. Existing...")

# retrieve information about an activation code
activationCodeStatus = licenseMngr.retrieveActivationCodeInfo(licenseServerId, activationCode)
print ("Activation code description and status: "+str(activationCodeStatus))

# activate the code
activationResult = licenseMngr.activateActivationCode(licenseServerId, activationCode, quantity=2)
print ("Activation result: "+str(activationCodeStatus))

serverActiveLicenses = licenseMngr.retrieveLicenses(activeServer['id'])
print("The following licenses are active: " + str(serverActiveLicenses))

# deactivate the code
activationResult = licenseMngr.activateActivationCode(licenseServerId, activationCode, quantity=2)
print ("Activation result: "+str(activationCodeStatus))

# disable logging
logger = logging.getLogger('bps_restpy')
logger.setLevel(logging.CRITICAL)

serverActiveLicenses = licenseMngr.retrieveLicenses(activeServer['id'])
print("The following licenses are active: " + str(serverActiveLicenses))

# retrieve license statistics. The amount of licenses used and available
licenseCountStats = licenseMngr.retrieveCountedFeatureStats(activeServer['id'])
print("The license statistics: " + str(licenseCountStats))


'''
K-Means聚类
1、	确定K值，即将数据聚集成K个类簇或小组
2、	从数据集中随机选择K个数据点作为质心（Centroid）或数据中心
3、	分别计算每个点到每个质心之间的距离，并将每个点划分到离最近质心的小组。
4、	当每个质心都聚集了一些点后，重新定义算法选出新的质心。（对于每个簇，计算其均值，即得到新的K个质心点）
5、	迭代执行第三步到第四步，直到迭代终止条件满足为止（聚类结果不再变化）
'''

from sklearn.cluster import KMeans

"""
第一部分：数据集
X表示二维矩阵数据，篮球运动员比赛数据
总共20行，每行两列数据
第一列表示玩家每局击杀数：assault_per_minute
第二列表示玩家每局死亡数：dead_per_minute
第三列表示玩家每局助攻数：assists_per_minute
"""
X = [[6,11,13],
     [10,11,10],
     [4,8,19],
     [9,11,10],
     [0,11,17],
     [6,4,24],
     [16,9,19],
     [8,6,29],
     [14,5,21],
     [7,5,24]
     ]

print(X)

#第二部分：KMeans聚类
clf = KMeans(n_clusters=3,n_init='auto')                                           # clf即赋值为KMeans；n_clusters=3) 表示类簇数为3，聚成3类数据；n_init='auto'是将KMeans改为自动运行模式，随着KM1.4版本更新，这样就能避免掉Futurewarning的报错
y_pred = clf.fit_predict(X)                                           # 载入数据集X，并且将聚类的结果赋值给y_pred

print(clf)                                                            # 输出完整Kmeans函数

print("y_pred = ", y_pred)                                            # 输出聚类预测结果

#第三部分：可视化绘图

import numpy as np
import matplotlib.pyplot as plt

x = [n[0] for n in X]                                                # 获取数据集的第一列//第三列 使用for循环获取 n[0]表示X第一列
print(x)
y = [n[2] for n in X]                                                # n[0]表示X的第三列
print(y)

'''
绘制散点图
参数：x横轴; y纵轴; c=y_pred聚类预测结果; marker类型:o表示圆点,*表示星型,x表示点;
'''
plt.scatter(x, y ,c=y_pred, marker='x')

plt.title("Kmeans-Gameplayer Data")                                      # 绘制标题

plt.xlabel("assault_per_minute")                                         # 绘制x轴和y轴坐标
plt.ylabel("assists_per_minute")

plt.legend(["A", "B", "C"])                                              # 设置右上角图例

plt.show()                                                               # 显示图形

from warnings import simplefilter

simplefilter("ignore", category=FutureWarning)


# csv 파일 크롤링하기

#ihongss.com/csv/exam1.csv

import csv
import pymongo

conn = pymongo.MongoClient('192.168.99.100', 32766)
db = conn.get_database("db1") # db1생성
coll = db.get_collection("exam1") # 테이블 생성

f = open('./resources/exam1.csv', 'r', encoding="utf-8")
rdr = csv.reader(f)
next(rdr, None) # 컬럼 skip

for line in rdr:
    print(type(line))
    print(line)
    dict1 = dict()
    dict1[line[0]] = line
    coll.insert_one(dict1)

conn.close
# DB에 넣는 방법


from django.urls import path
from . import views

urlpatterns = [
    path('', views.cart, name='cart'),
    path('add_to_cart', views.add_to_cart, name='add_to_cart'),
    path('remove_item', views.remove_item, name='remove_item'),
    path('check_out', views.check_out, name='check_out'),
    path('customr_details', views.customer_details, name='customer_details'),
    path('clear_cart/<int:cart_id>', views.clear_cart, name='clear_cart'),

]


from flask import Flask, render_template, request, make_response
app = Flask(__name__)

@app.route('/cookie')
def index():
   return render_template('cookie_index.html')

@app.route('/setcookie', methods = ['POST', 'GET'])
def setcookie():
    if request.method == 'POST':
        user = request.form['nm']

        #get response object from return value of a view function
        resp = make_response(render_template('readcookie.html',cook =user))
        #sets a Cookie name userID and form value.
        resp.set_cookie('userID', user)

        return resp

@app.route('/getcookie')
def getcookie():
   #reads back and displays the cookie value in browser
   name = request.cookies.get('userID')
   return '
welcome '+name+'
'

if __name__ == '__main__':
    app.run(debug=True)


import argparse
import logging
import os
import sys
import urllib.error
from pathlib import Path

import numpy as np

path_mod = f"{Path(os.path.dirname(os.path.realpath(__file__))).parents[1]}/"
sys.path.append(path_mod)
from src.utils_general.utils import download_ftp, download_url, unzip

logger = logging.getLogger(__name__)

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("iso3", help="iso3 code of country of interest")
    parser.add_argument("-a", "--admin_level", default=1)
    # Prefix for filenames
    parser.add_argument(
        "-s",
        "--suffix",
        default="",
        type=str,
        help="Suffix for output files, and if applicable input files",
    )
    parser.add_argument(
        "-d",
        "--download-data",
        action="store_true",
        help=(
            "Download the raw data. FewsNet and WorldPop are currently"
            " implemented"
        ),
    )
    parser.add_argument(
        "-da",
        "--dates",
        default=None,
        nargs="+",
        type=str,
        help="List of strings of dates to be included in YM format",
    )
    return parser.parse_args()

def download_fewsnet(date, iso2_code, region, regioncode, output_dir):
    """Retrieve the raw fewsnet data.

    Depending on the region, this date is published per region or per
    country. This function tries to retrieve both. The download_url
    always downloads the given url, but sometimes this doesn't return a
    valid zip file. This means that data doesn't exist. This happens
    often, since for most countries the classifications are on earlier
    dates only published per region and later on per country. This is
    not bad, and the function will remove the invalid zip files Args:
    date: str in yyyymm format. Date for which to retrieve the fewsnet
    data. iso2_code: iso2 code of the country of interest region: region
    that the fewsnet data covers, e.g. "east-africa" regioncode:
    abbreviation of the region that the fewsnet data covers, e.g. "EA"
    output_dir: directory to save the files to
    """
    FEWSNET_BASE_URL_REGION = "https://fews.net/data_portal_download/download?data_file_path=http%3A//shapefiles.fews.net.s3.amazonaws.com/HFIC/"  # noqa: E501
    FEWSNET_BASE_URL_COUNTRY = "https://fdw.fews.net/api/ipcpackage/"
    url_country = f"{FEWSNET_BASE_URL_COUNTRY}?country_code={iso2_code.upper()}&collection_date={date[:4]}-{date[-2:]}-01"  # noqa: E501
    zip_filename_country = os.path.join(output_dir, f"{iso2_code}{date}.zip")
    output_dir_country = os.path.join(output_dir, f"{iso2_code}{date}")
    if not os.path.exists(output_dir_country):
        # var to check if country data exists
        country_data = False
        try:
            download_url(url_country, zip_filename_country)
        except Exception:
            logger.warning(
                f"Url of country level FewsNet data for {iso2_code}, {date} is"
                " not valid"
            )
        try:
            unzip(zip_filename_country, output_dir_country)
            logger.info(
                f'Downloaded "{url_country}" to "{zip_filename_country}'
            )
            logger.info(f"Unzipped {zip_filename_country}")
            country_data = True
        except Exception:
            # indicates that the url returned something that wasn't a
            # zip, happens often and indicates data for the given
            # country - date is not available
            logger.info(
                f"No country level FewsNet data for {iso2_code}, {date}, using"
                " regional data if available"
            )
        os.remove(zip_filename_country)

        url_region = (
            f"{FEWSNET_BASE_URL_REGION}{regioncode}/{region}{date}.zip"
        )
        zip_filename_region = os.path.join(output_dir, f"{region}{date}.zip")
        output_dir_region = os.path.join(output_dir, f"{region}{date}")
        if not country_data and not os.path.exists(output_dir_region):
            try:
                if not os.path.exists(zip_filename_region):
                    download_url(url_region, zip_filename_region)
            except Exception:
                logger.warning(
                    f"Url of regional level FewsNet data for {region}, {date}"
                    " is not valid"
                )
            try:
                unzip(zip_filename_region, output_dir_region)
                logger.info(
                    f'Downloaded "{url_region}" to "{zip_filename_region}'
                )
                logger.info(f"Unzipped {zip_filename_region}")
            except Exception:
                # indicates that the url returned something that wasn't
                # a zip, happens often and indicates data for the given
                # country - date is not available
                logger.warning(
                    f"No FewsNet data for date {date} found that covers"
                    f" {iso2_code}"
                )
            os.remove(zip_filename_region)

def download_worldpop(country_iso3, year, output_dir, config):
    # create directory if doesn't exist
    Path(output_dir).mkdir(parents=True, exist_ok=True)
    url = config.WORLDPOP_URL.format(
        country_iso3_upper=country_iso3.upper(),
        country_iso3_lower=country_iso3.lower(),
        year=year,
    )
    output_file = os.path.join(output_dir, url.split("/")[-1])
    if not os.path.exists(output_file):
        try:
            download_ftp(url, output_file)
        except urllib.error.URLError as e:
            logger.warning(
                f"{e}. Data of the year of interest might not exist on the"
                " WorldPop FTP."
            )

def fewsnet_validperiod(row):
    """
    Add the period for which FewsNet's projections are valid. Till 2016
    FN published a report 4 times a year, where each projection period
    had a validity of 3 months From 2017 this has changed to thrice a
    year, where each projection period has a validity of 4 months Args:
    row: row of dataframe containing the date the FN data was published
    (i.e. CS period) as timestamp

    Returns:

    """
    # make own mapping, to be able to use mod 12 to calculate months of
    # projections
    month_abbr = {
        1: "Jan",
        2: "Feb",
        3: "Mar",
        4: "Apr",
        5: "May",
        6: "Jun",
        7: "Jul",
        8: "Aug",
        9: "Sep",
        10: "Oct",
        11: "Nov",
        0: "Dec",
    }
    year = row["date"].year
    month = row["date"].month
    if year <= 2015:
        if month == 10:
            year_ml2 = year + 1
        else:
            year_ml2 = year
        period_ML1 = (
            f"{month_abbr[month]} - {month_abbr[(month + 2) % 12]} {year}"
        )
        period_ML2 = (
            f"{month_abbr[(month + 3) % 12]} - {month_abbr[(month + 5) % 12]}"
            f" {year_ml2}"
        )
    if year > 2015:
        if month == 2:
            year_ml1 = year
            year_ml2 = year
        elif month == 6:
            year_ml1 = year
            year_ml2 = year + 1
        elif month == 10:
            year_ml1 = year + 1
            year_ml2 = year + 1
        else:
            logger.info(
                "Period of ML1 and ML2 cannot be added for non-regular"
                f" publishing date {year}-{month}. Add manually."
            )
            row["period_ML1"] = np.nan
            row["period_ML2"] = np.nan
            return row
        period_ML1 = (
            f"{month_abbr[month]} - {month_abbr[(month + 3) % 12]} {year_ml1}"
        )
        period_ML2 = (
            f"{month_abbr[(month + 4) % 12]} - {month_abbr[(month + 7) % 12]}"
            f" {year_ml2}"
        )
    row["period_ML1"] = period_ML1
    row["period_ML2"] = period_ML2
    return row

def compute_percentage_columns(df, config):
    """
    calculate percentage of population per analysis period and level
    Args: df (pd.DataFrame): input df, should include columns of the
    IPC_PERIOD_NAMES for eah period in range(1,6) config (Config):
    food-insecurity config class Returns: df(pd.DataFrame): input df
    with added percentage columns
    """
    for period in config.IPC_PERIOD_NAMES:
        # IPC level goes up to 5, so define range up to 6
        for i in range(1, 6):
            c = f"{period}_{i}"
            df[f"perc_{c}"] = df[c] / df[f"pop_{period}"] * 100
        # get pop and perc in IPC3+ and IPC2-
        # 3p = IPC level 3 or higher, 2m = IPC level 2 or lower
        df[f"{period}_3p"] = df[[f"{period}_{i}" for i in range(3, 6)]].sum(
            axis=1
        )
        df[f"perc_{period}_3p"] = (
            df[f"{period}_3p"] / df[f"pop_{period}"] * 100
        )
        df[f"{period}_4p"] = df[[f"{period}_{i}" for i in range(4, 6)]].sum(
            axis=1
        )
        df[f"perc_{period}_4p"] = (
            df[f"{period}_4p"] / df[f"pop_{period}"] * 100
        )
        df[f"{period}_2m"] = df[[f"{period}_{i}" for i in range(1, 3)]].sum(
            axis=1
        )
        df[f"perc_{period}_2m"] = (
            df[f"{period}_2m"] / df[f"pop_{period}"] * 100
        )
    df["perc_inc_ML2_3p"] = df["perc_ML2_3p"] - df["perc_CS_3p"]
    df["perc_inc_ML1_3p"] = df["perc_ML1_3p"] - df["perc_CS_3p"]
    # round the percentage columns to 3 decimals
    perc_cols = [col for col in df.columns if "perc" in col]
    df[perc_cols] = df[perc_cols].round(3)
    return df


#!/usr/bin/env python3

import argparse
import rospy
from PIL import Image
import numpy as np
from geometry_msgs.msg import PoseStamped
from geometry_msgs.msg import Twist

class PotentialFields:
    def __init__(self,robot_name):
        rospy.init_node('potential_fields_node')
        self.pixels_array = self.obstacles("map_1.png")
        self.posicao_atual = PoseStamped()
        self.posicao_atual.header.frame_id = 'map'  # Define o frame de referência
        self.posicao_atual.pose.position.x = 5.0  # Define a posição x inicial como 5
        self.posicao_atual.pose.position.y = 5.0  # Define a posição y inicial como 5
        self.posicao_atual.pose.position.z = 0.0  # Define a posição z inicial como 0

        self.position_publisher = rospy.Publisher(f'/{robot_name}/current_position', PoseStamped, queue_size=1)
        self.twist_publisher = rospy.Publisher(f'/{robot_name}/cmd_vel', Twist, queue_size=1)

    def obstacles(self, map):

        image = Image.open(f"src/tcc/worlds/{map}")
        self.image_gray = image.convert('L')
        self.pixels_array = np.array(self.image_gray)
        self.threshold = 254
        filled_pixels = self.pixels_array < self.threshold
        return filled_pixels

    def potential_fields(self):
        print(self.posicao_atual)
        ponto_destino = (5, 5)  # Substitua com as coordenadas do ponto de destino
        vetor_atracao = ponto_destino - np.array([self.posicao_atual.pose.position.x, self.posicao_atual.pose.position.y])

        vetor_repulsao_total = np.zeros(2)  # Inicialize com vetor nulo
        constante_repulsao = 1000  # Ajuste conforme necessário

        for i in range(self.pixels_array.shape[0]):
            for j in range(self.pixels_array.shape[1]):
                if self.pixels_array[i, j]:
                    vetor_repulsao = np.array([j, i]) - np.array([self.posicao_atual.pose.position.x, self.posicao_atual.pose.position.y])
                    magnitude = constante_repulsao / np.linalg.norm(vetor_repulsao)**2
                    vetor_repulsao_total += vetor_repulsao / np.linalg.norm(vetor_repulsao) * magnitude

        vetor_resultante = vetor_atracao + vetor_repulsao_total

        self.posicao_atual.pose.position.x += vetor_resultante[0]
        self.posicao_atual.pose.position.y += vetor_resultante[1]

        # Publica a posição atual
        self.position_publisher.publish(self.posicao_atual)

        # Enviar comandos para o robô (exemplo hipotético)
        velocidade_linear = 0.1  # Ajuste conforme necessário
        velocidade_angular = 0.0  # Ajuste conforme necessário

        # Comande o movimento do robô (depende do seu ambiente e tipo de robô)
        # Exemplo hipotético:
        self.move_robot(velocidade_linear, velocidade_angular)

    def move_robot(self, linear, angular):
        comando = Twist()
        comando.linear.x = linear
        comando.angular.z = angular
        print("comando", comando)
        self.twist_publisher.publish(comando)

    def run(self):
        rate = rospy.Rate(10)

        while not rospy.is_shutdown():
            self.potential_fields()
            rate.sleep()

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Control a human-like robot.')
    parser.add_argument('robot_name', type=str, help='Name of the robot')
    args = parser.parse_args()
    try:
        pf = PotentialFields(args.robot_name)
        pf.run()
    except rospy.ROSInterruptException:
        pass


import os

os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
import re
from paddleocr import PaddleOCR, draw_ocr
import cv2 as cv
import numpy as np
import time
from collections import defaultdict
import json
from fuzzywuzzy import fuzz
from fuzzywuzzy import process

BASE_DIR = os.path.dirname(os.path.abspath(__file__))

keyword1 = "goal"

def makedir(new_dir):
    if not os.path.exists(new_dir):
        os.makedirs(new_dir)

EVENT_DIR = "D:/dataset/event"
VIDEO_DIR = "D:/dataset/video_error"

ocr = PaddleOCR(lang="en", gpu_mem=5000, det=False,
                rec_model_dir="./inference/en_ppocr_mobile_v2.0_rec_infer/")  # 首次执行会自动下载模型文件

# 获取队名文件
team_name = []
f = open("team_name.txt", "r")
for line in f:
    line = line[:-1]
    team_name.append(line)

# result = process.extract(a, team_name, scorer=fuzz.token_set_ratio, limit=5)

def get_ocr_result(videoCap, i, h_index):
    result = ''
    ocr_list = []
    temp_result = []

    videoCap.set(cv.CAP_PROP_POS_FRAMES, i)
    # i += 100;
    boolFrame, matFrame = videoCap.read()

    if boolFrame:
        temp_jpgframe = np.asarray(matFrame)
        # 截取上面0-90区域进行OCR检测
        if h_index < 200:
            jpgframe = temp_jpgframe[0:h_index]
        else:
            jpgframe = temp_jpgframe[h_index:]

        # 得到OCR识别结果
        temp_result = ocr(jpgframe)

        for mystr, ration in temp_result[1]:
            result += mystr
            ocr_list.append(mystr)
            result += ' '
    return result, temp_result

def find_goal_location(temp_result, location, init_candidate, gt_y2):
    goal_flag = False
    tempResult_index = 0
    for ii, [strr, ration] in enumerate(temp_result[1]):
        game_time = re.findall("\d:\d", strr)
        if game_time:
            num_num = game_time[0]
            # 如果已经记录过位置，则与记录位置进行比较
            if location:
                # x1 = temp_result[0][ii][0][0]
                # x2 = temp_result[0][ii][1][0]
                # y1 = temp_result[0][ii][0][1]
                y2 = temp_result[0][ii][2][1]
                # 偏移量小于8则说明位置正确
                # if (abs(gt_x1 - x1) + abs(gt_x2 - x2) + abs(gt_y1 - y1) + abs(gt_y2 - y2) < 8):
                if abs(gt_y2 - y2) < 4:
                    num1, num2 = num_num.split(":")
                    init_candidate[num1 + "-" + num2] += 1
                    # tempResult_index = ii
                    goal_flag = True
                    break
            # 没记录过，则只记录在init_candidate
            else:
                num1, num2 = num_num.split(":")
                init_candidate[num1 + "-" + num2] += 1
                # 记录该字符串在temp_result中的索引
                tempResult_index = ii
                goal_flag = True
                break
    return init_candidate, goal_flag, tempResult_index

def get_frames(video_dir):
    get = 0
    miss = 0
    total_num = 0
    more = 0
    # with open('video_6_score_change.json', 'r') as fd:
    #     Video_Score_dic = json.load(fd)
    SCORES = {}

    for root, _, files in os.walk(video_dir):
        for video_index in files:
            time1 = time.time()
            video_index1 = video_index.split('_')

            if video_index1[1] not in ['0059', '0060', '0061', '0062',
                                       '0063', '0064', '0065', '0066', '0067', '0068', '0069', '0070',
                                       '0071', '0072', '0073', '1040', '1041', '1045', '1046', '1047', '1048',
                                       '1049', '1050', '1051', '1052', '1054',
                                       '1055', '1056', '1057', '1058', '1059', '1171',
                                       '1212', '1216', '1218', '1221', '1223', '1224', '1225',
                                       '1226', '1228', '1230', '1231', '1233', '1236', '1237', '1238', '1239', '1242',
                                       '1243', '1244', '1245']:

                score_dic = {}
                score_record = []
                player_score_dic = {}
                player_score_record = []
                videoCap = cv.VideoCapture(video_dir + "\\" + video_index)

                frame_height = videoCap.get(cv.CAP_PROP_FRAME_HEIGHT)
                frame_width = videoCap.get(cv.CAP_PROP_FRAME_WIDTH)
                big_h_index = int(frame_height * 7 / 10)
                big_w_index = int(frame_width / 2)
                x1_big_center = big_w_index - 50
                x2_big_center = big_w_index + 50

                frame_count = videoCap.get(cv.CAP_PROP_FRAME_COUNT)
                print("-----------------------------")
                print("video:{}".format(video_index1[1]))
                i = frame_count - 10000
                init_candidate = defaultdict(int)
                player_name_dic = defaultdict(int)
                score_time_dic = defaultdict(int)
                big_candidate = defaultdict(set)

                location_player_score = 0  # 得分球员位置，为0代表在队名下面，为1代表在队名上面
                tempResult_index = 0
                mode = 0
                modify_candidate = defaultdict(int)
                temp_result = []
                gt_y2 = 0
                left_team = ''
                right_team = ''
                team_y1 = 10000
                team_y2 = 0
                big_flag = False
                big_nums = 6
                while i < frame_count:
                    # 评估候选比分

                    # 获取ocr第i帧的直接结果temp_result， 以及字符串连接后的result
                    result, temp_result = get_ocr_result(videoCap, i, 125)
                    big_result, big_temp_result = get_ocr_result(videoCap, i, big_h_index)

                    goal_flag = False
                    big_flag = False
                    team_nums = 0

                    if big_result:
                        # 首先检测是否存在两个队名
                        for ii, [strr, ration] in enumerate(big_temp_result[1]):
                            x1_big = big_temp_result[0][ii][0][0]
                            x2_big = big_temp_result[0][ii][2][0]
                            y1_big = big_temp_result[0][ii][0][1]
                            y2_big = big_temp_result[0][ii][3][1]
                            team1 = process.extractOne(strr, team_name, scorer=fuzz.token_set_ratio,
                                                       score_cutoff=70)
                            if team1:
                                team_nums += 1
                                if x1_big > big_w_index:
                                    right_team = team1[0]
                                if x2_big < big_w_index:
                                    left_team = team1[0]
                                team_y1 = y1_big
                                team_y2 = y2_big
                                continue

                        # 检测出了两个球队名，说明是大字幕，此时team_y1，team_y2都有了
                        if team_nums == 2:
                            for ii, [strr, ration] in enumerate(big_temp_result[1]):
                                x1_big = big_temp_result[0][ii][0][0]
                                x2_big = big_temp_result[0][ii][2][0]
                                y1_big = big_temp_result[0][ii][0][1]
                                y2_big = big_temp_result[0][ii][3][1]

                                if abs(y1_big - team_y1) + abs(y2_big - team_y2) < 4:
                                    continue

                                # 首先排除时间和比分的数字影响
                                sc = re.findall("\d-\d", strr)
                                if sc:
                                    # x1_big_goal = big_temp_result[0][ii][0][0]
                                    # x2_big_goal = big_temp_result[0][ii][3][0]
                                    continue
                                game_t = re.findall("\d\d:\d\d", strr)
                                if game_t:
                                    continue

                                # 运动员进球信息格式1：
                                # 1个或多个非数字+1个或多个数字+0个或多个非数字 |
                                # 0个或多个非数字+1个或多个数字+1个或多个非数字
                                # 长度要求大于4
                                player_score_list = re.findall("\D+\d+\D*|\D*\d+\D+", strr)
                                if player_score_list and len(player_score_list[0]) > 4:
                                    player_score_temp = player_score_list[0]

                                    string_list = re.findall("[A-Z][a-z]+", player_score_temp)
                                    number_list = re.findall("\d+", player_score_temp)

                                    if not string_list:
                                        continue
                                    player_name = string_list[0]
                                    player_name_dic[player_name] += 1

                                    for number_one in number_list:
                                        if len(number_one) < 4:
                                            big_candidate[player_name].add(number_one)
                                            score_time_dic[number_one] += 1
                                            big_flag = True
                                            print("check players name: {}  goal_time: {}  frame:{}   time:{}".format(
                                                player_name, big_candidate[player_name], i,
                                                game_t))
                                    continue

                                # 运动员进球信息格式2：
                                # 运动员字符串在数字字符串前一个
                                #
                                # goal_time = re.findall("^\d+$|^\d+\'|^\d+\(", strr)
                                number_list = re.findall("\d+", strr)

                                if number_list and team_y2 and (team_y1 - y1_big > 5 or y2_big - team_y2 > 5):
                                    for jj, [sstr, ration] in enumerate(big_temp_result[1]):
                                        y1_big = big_temp_result[0][jj][0][1]
                                        y2_big = big_temp_result[0][jj][3][1]

                                    player_name, rat = big_temp_result[1][ii - 1]
                                    player_name_dic[player_name] += 1
                                    isTeam = process.extractOne(player_name, team_name, scorer=fuzz.token_set_ratio,
                                                                score_cutoff=70)
                                    if isTeam:
                                        continue

                                    number_one = re.findall("\d+", goal_time[0])

                                    for number_one in number_list:
                                        if len(number_one) < 4:
                                            big_candidate[player_name].add(number_one)
                                            score_time_dic[number_one] += 1
                                            big_flag = True
                                            print("check players name: {}  goal_time: {}  frame:{}   time:{}".format(
                                                player_name, big_candidate[player_name], i,
                                                game_t))
                        # if big_flag:
                        #     big_nums += 1
                        # elif big_nums > 2 and not big_flag:
                        #     big_nums = 1

                    # 如果检测出比分，则帧数+1检测下一帧，否则帧数+200
                    if goal_flag:
                        i += 1
                        mode = 2
                        continue
                    elif big_flag:
                        i += 2
                        big_nums += 5
                        continue
                    # 没有大字幕持续若干次
                    elif big_nums < 4 and not big_flag:
                        big_nums += 1
                        i += 2
                        continue
                    # 从有大字幕突然变成没有大字幕
                    elif big_nums > 6 and not big_flag:
                        big_nums = 1
                        i += 5
                        continue
                    # 没有大字幕持续若干次后，进行跨越，再检测
                    elif big_nums == 4:
                        # if i+10000 < frame_count-5000:
                        #     i += 10000
                        # else:
                        #     i = max(i+1000, frame_count-5000)
                        if i + 3000 < frame_count - 2000:
                            i += 3000
                        else:
                            i = max(i + 2000, frame_count - 2000)
                        big_nums = 6
                        continue
                    else:
                        i += 50
                        # print("can not get frames!")
                        continue

                # 把big_candidate里的值（set）取交集，如果不为空，就在player_name_dic里面看谁的key次数多
                # 次数少的key就在big_candidate中删除
                nn = len(big_candidate)
                names = list(big_candidate.keys())
                score_times = list(score_time_dic.keys())
                for i in range(nn):
                    if big_candidate[names[i]] != set('0'):
                        for score_time in list(big_candidate[names[i]]):
                            if score_time_dic[score_time] < 5:
                                big_candidate[names[i]].remove(score_time)
                        if big_candidate[names[i]] == set():
                            big_candidate[names[i]] = set('0')
                        for j in range(i + 1, nn):
                            if big_candidate[names[j]] != set('0'):
                                set_temp = big_candidate[names[i]] & big_candidate[names[j]]
                                if set_temp:
                                    if player_name_dic[names[i]] >= player_name_dic[names[j]]:
                                        big_candidate[names[j]] = set('0')
                                    else:
                                        big_candidate[names[i]] = set('0')
                                elif fuzz.token_set_ratio(names[i], names[j]) >= 70:
                                    big_candidate[names[i]] = big_candidate[names[i]].union(big_candidate[names[j]])
                                    big_candidate[names[i]] = set('0')

                # big_candidate里的值（set）为‘0’的key也应该删除
                for key in names:
                    if '0' in big_candidate[key]:
                        del big_candidate[key]
                    elif key.isdigit():
                        del big_candidate[key]
                    elif re.findall("\d\d", key):
                        del big_candidate[key]

                print(big_candidate)
                time2 = time.time()
                print("time for this video:{}".format(time2 - time1))

                # SCORES[video_index] = score_record

                # with open('scores_3' + '.json', 'w') as fd:
                #     json.dump(SCORES, fd)

    print("total_num:{}   get:{}   miss:{}  more:{}".format(total_num, get, miss, more))

get_frames(video_dir=VIDEO_DIR)


from urllib.request import urlopen
import time
import json
import os
import re

from plant_fba.core.fetch_plantseed_impl import FetchPlantSEEDImpl

PS_url = 'https://raw.githubusercontent.com/ModelSEED/PlantSEED/'
PS_tag = 'v3.0.0'

# These should be retrieved from the Template data
Template_Compartment_Mapping={'c':'cytosol', 'g':'golgi', 'w':'cellwall',
                              'n':'nucleus', 'r':'endoplasm',
                              'v':'vacuole', 'cv':'vacuole',
                              'd':'plastid', 'cd':'plastid',
                              'm':'mitochondria','cm':'mitochondria',
                              'mj':'mitointer', 'ce':'extracellular',
                              'x':'peroxisome', 'cx':'peroxisome',
                              'e':'extracellular','de':'plastid'}

class GenerateTableImpl:

    def log(self,message, prefix_newline=False):
        time_str = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(time.time()))
        print(('\n' if prefix_newline else '') + time_str + ': ' + message)

    def __init__(self):
        pass

    def generate_table(self, reactions, complexes=None):

        html_lines=list()
        html_lines.append('
')

        header_list = ["Reaction","Compartment","Roles","Enzymes","EC numbers","Subsystems","Classes"]

        html_lines.append('
')
        internal_header_line = "	".join(header_list)
        html_lines.append('
'+internal_header_line+'
')
        html_lines.append('
')

        html_lines.append("
")

        #######################################################################
        # Each row in the table is unique to the reaction/compartment combination
        for reaction in sorted(reactions.keys()):
            for compartment in reactions[reaction]['compartments']:
                compartment_str = Template_Compartment_Mapping[compartment]

                compartmentalized_complexes=list()

                #######################################################################
                # The code below is taken from the code for generating a ModelTemplate
                # It identifies the single letter reaction id for transport reactions
                # Accordingly, the compartments should all be sorted
                # So a compartment index of 0 matches the first position in the compartment list
                # The order is curated in the PlantSEED database

                reaction_cpt_id = compartment

                # If its a transporter, need to update the reaction compartment id
                if(len(compartment)==2):

                    # The rule is that it is always the non-cytosolic compartment
                    if('c' in compartment):
                        for cpt in compartment:
                            if(cpt != 'c'):
                                reaction_cpt_id = cpt

                    # With two main exceptions:
                    # 1) whether its an extracellular transporter
                    if('e' in compartment):
                        for cpt in compartment:
                            if(cpt != 'e'):
                                reaction_cpt_id = cpt

                    # 2) whether its an intraorganellar transporter
                    if('j' in compartment):
                        reaction_cpt_id = 'j'

                #######################################################################

                model_reaction_id = reaction+"_"+reaction_cpt_id+"0"
                if(complexes is not None and \
                       model_reaction_id in complexes and \
                       complexes[model_reaction_id] not in compartmentalized_complexes):
                    compartmentalized_complexes.append(complexes[model_reaction_id])

                complexes_str="; ".join(compartmentalized_complexes)
                roles_str="; ".join(sorted(reactions[reaction]['roles']))
                ecs_str="; ".join(sorted(reactions[reaction]['ecs']))
                classes_str="; ".join(sorted(reactions[reaction]['classes']))

                subsystems_str ="; ".join(sorted(reactions[reaction]['subsystems']))
                subsystems_str = subsystems_str.replace("_in_plants","")
                subsystems_str = subsystems_str.replace("_"," ")

                html_lines.append("
")
                internal_row_line = "	".join([reaction,compartment_str,roles_str,
                                                      complexes_str,ecs_str,subsystems_str,classes_str])
                html_lines.append(""+internal_row_line+"	")
                html_lines.append("
")

        html_lines.append("
")
        html_lines.append("
")

        return "\n".join(html_lines)

def main():

    # Load these directly from PlantSEED_Roles.json
    PS_Roles = json.load(urlopen(PS_url+PS_tag+'/Data/PlantSEED_v3/PlantSEED_Roles.json'))

    plantseed = FetchPlantSEEDImpl()
    reactions_data = plantseed.fetch_reactions(PS_Roles)

    table = GenerateTableImpl()
    table_html_string = table.generate_table(reactions_data)

    with open(os.path.join('../../../data','app_report_templates',
                           'integrate_abundances_report_tables_template.html')) as report_template_file:
        report_template_string = report_template_file.read()

    # Insert html table
    table_report_string = report_template_string.replace('*TABLES*', table_html_string)

    table_html_file="annotation_table_output.html"
    with open(os.path.join('../../../data','app_report_templates',
                           table_html_file),'w') as table_file:
        table_file.write(table_report_string)

    table.log(message="Table written to file: "+table_html_file)

if(__name__ == "__main__"):
    main()


# https://leetcode.com/problems/moving-average-from-data-stream/description/

"""
Given a stream of integers and a window size,
calculate the moving average of all integers in the sliding window.

Implement the MovingAverage class:
    * MovingAverage(int size) Initializes the object with the size of the window size.

    * double next(int val) Returns the moving average of the last size values of the stream.

"""

from collections import deque
from utils import display_func_value

class MovingAverage:

    def __init__(self, size: int):
        self.current_ma_value = 0
        self.data = deque()
        self.size = size

    @display_func_value
    def next(self, val: int) -> float:
        self.data.appendleft(val)
        self.current_ma_value += val

        # достигли размера окна (size) => нужно удалить самый первый элемент очереди
        if len(self.data) > self.size:
            self.current_ma_value -= self.data.pop()

        return self.current_ma_value / min(len(self.data), self.size)

movingAverage = MovingAverage(3)
movingAverage.next(1)  # return 1.0 = 1 / 1
movingAverage.next(10) # return 5.5 = (1 + 10) / 2
movingAverage.next(3)  # return 4.66667 = (1 + 10 + 3) / 3
movingAverage.next(5)  # return 6.0 = (10 + 3 + 5) / 3

Problems Solutions/346. Moving Average from Data Stream.py


from bokeh.layouts import layout, row, column
from bokeh.plotting import save, figure, output_file, show, Column
from bokeh.models.glyphs import Text, ImageURL
from bokeh.models import DataTable, TableColumn, PointDrawTool, ColumnDataSource,Arrow, OpenHead, NormalHead, VeeHead, HoverTool, Div
from bokeh.models.widgets.buttons import Button
from bokeh.models import Tabs, Panel
from bokeh.models.widgets import Slider,Select,PreText,Dropdown,TextInput,AutocompleteInput,CheckboxGroup,RadioGroup,RangeSlider
from bokeh.transform import cumsum
from scipy.misc import imread
from bokeh.io import curdoc
import pandas as pd
import numpy as np

########### Dropbdown countries ###########

def check_dropdown_update_country(attr, old, new):
	global confirmed
	global panels
	global deaths
	global recovered
	global dates
	global tot_confirmed,tot_recovered,tot_recovered, data_df

	if dropdown_countries.value == "World":
		tot_confirmed = [sum(confirmed[d]) for d in dates]
		tot_deaths = [sum(deaths[d]) for d in dates]
		tot_recovered = [sum(recovered[d]) for d in dates]
	else:
		reg = dropdown_countries.value.split(": ")
		if (len(reg) == 1):
			confirmed_filter = confirmed[confirmed['Country/Region'] == dropdown_countries.value]
			deaths_filter = deaths[deaths['Country/Region'] == dropdown_countries.value]
			recovered_filter = recovered[recovered['Country/Region'] == dropdown_countries.value]
		else:
			confirmed_filter = confirmed[(confirmed['Country/Region'] == reg[0]) & (confirmed['Province/State'] == reg[1])]
			deaths_filter = deaths[(deaths['Country/Region'] == reg[0]) & (deaths['Province/State'] == reg[1])]
			recovered_filter = recovered[(recovered['Country/Region'] == reg[0]) & (recovered['Province/State'] == reg[1])]
		tot_confirmed = [sum(confirmed_filter[d]) for d in dates]
		tot_deaths = [sum(deaths_filter[d]) for d in dates]
		dates_recovered = list(recovered.columns[4:len(recovered.columns)])
		tot_recovered = [sum(recovered_filter[d]) for d in dates_recovered]
		if len(tot_recovered) != len(tot_confirmed):
			tot_recovered = tot_recovered + ["-"]

	panels = []
	for axis_type in ["linear", "log"]:
		p = figure(plot_width=1000, plot_height=550, y_axis_type = axis_type, x_axis_type="datetime", title="COVID-19 cumulative data (@kpelechrinis) ")
		panel = Panel(child=p, title=axis_type)
		panels.append(panel)
		tabs = Tabs(tabs=panels)
		data_df = pd.DataFrame({'Date':dates, 'Confirmed': tot_confirmed, 'Deaths': tot_deaths, 'Recovered': tot_recovered})
		data_df['Date'] = pd.to_datetime(data_df['Date'])
		p.line(x=data_df['Date'], y=tot_confirmed, line_width=4, color="red", alpha=0.8,legend="Confirmed")
		p.line(x=data_df['Date'], y=tot_deaths, line_width=4, color="black", alpha=0.8,legend="Deaths")
		p.line(x=data_df['Date'], y=tot_recovered, line_width=4, color="green", alpha=0.8,legend="Recovered")
		p.legend.location = "top_left"
		nsource = ColumnDataSource({
			'Date': list(dates), 'Confirmed': list(data_df['Confirmed']), 'Deaths': list(data_df['Deaths']), 'Recovered': list(data_df['Recovered'])
		})
	columns = [TableColumn(field="Date", title="Date"),
		TableColumn(field="Confirmed", title="Confirmed"),
		TableColumn(field="Deaths", title="Deaths"),
		TableColumn(field="Recovered",title="Recovered")]
	table = DataTable(source=nsource, columns=columns, editable=True, height=550)
	layout.children[1] = tabs
	layout.children[3] = table

global confirmed
global deaths
global recovered
global panels
global dates
global tot_confirmed,tot_deaths,tot_recovered,data_df

#confirmed = pd.read_csv("https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Confirmed.csv")
#deaths = pd.read_csv("https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Deaths.csv")
#recovered = pd.read_csv("https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Recovered.csv")

confirmed = pd.read_csv("https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_confirmed_global.csv")
deaths = pd.read_csv("https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_deaths_global.csv")
recovered = pd.read_csv("https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_recovered_global.csv")

dates = list(confirmed.columns[4:len(confirmed.columns)])
tot_confirmed = [sum(confirmed[d]) for d in dates]
tot_deaths = [sum(deaths[d]) for d in dates]
dates_recovered = list(recovered.columns[4:len(recovered.columns)])
tot_recovered = [sum(recovered[d]) for d in dates_recovered]

if len(tot_recovered) != len(tot_confirmed):
	tot_recovered = tot_recovered + ["-"]

panels = []

for axis_type in ["linear", "log"]:
	p = figure(plot_width=1000, plot_height=550, y_axis_type = axis_type, x_axis_type="datetime", title="COVID-19 cumulative data (@kpelechrinis)")
	panel = Panel(child=p, title=axis_type)
	panels.append(panel)
	tabs = Tabs(tabs=panels)
	data_df = pd.DataFrame({'Date':dates, 'Confirmed': tot_confirmed, 'Deaths': tot_deaths, 'Recovered': tot_recovered})
	data_df['Date'] = pd.to_datetime(data_df['Date'])
	p.line(x=data_df['Date'], y=data_df['Confirmed'],  line_width=4, color="red", alpha=0.8, legend="Confirmed")
	p.line(x=data_df['Date'], y=data_df['Deaths'],  line_width=4, color="black", alpha=0.8, legend="Deaths")
	p.line(x=data_df['Date'], y=data_df['Recovered'],  line_width=4, color="green", alpha=0.8, legend="Recovered")
	p.legend.location = "top_left"

	source = ColumnDataSource({
		'Date': list(dates), 'Confirmed': list(data_df['Confirmed']), 'Deaths': list(data_df['Deaths']), 'Recovered': list(data_df['Recovered'])
	})

columns = [TableColumn(field="Date", title="Date"),
           TableColumn(field="Confirmed", title="Confirmed"),
           TableColumn(field="Deaths", title="Deaths"),
           TableColumn(field="Recovered",title="Recovered")]

table = DataTable(source=source, columns=columns, editable=True, height=550)

#############################
logo = imread("pitt-sci.png")
## there is a bug in bokeh image_rgba to be used later that requires the following flipping
## https://github.com/bokeh/bokeh/issues/1666
logo = logo[::-1]
plogo = figure(x_range=(0,25),y_range=(0,15), tools = [], plot_width=450,plot_height=350)
plogo.xgrid.grid_line_color = None
plogo.ygrid.grid_line_color = None
plogo.image_rgba(image=[logo],x=[0], y=[0], dw = [25], dh = [15])
plogo.xaxis.visible = False
plogo.yaxis.visible = False

div1 = Div(text="""<h>Coronavirus data</h>

This is a simple time-series visualization of the confirmed cases, deaths and recoveries from the coronavirus according to the Johns Hopkins dataset that is updated daily. The data are updated automatically (approximately once every day).

Visualization from, K. Pelechrinis (@kpelechrinis) - School of Computing and Information, University of Pittsburgh.

""",width=800, height=250)

menu_countries = ["World"]+list(np.sort(list(confirmed['Country/Region'].unique())))
menu_china = list(np.sort(list(confirmed[confirmed['Country/Region'] == "China"]['Province/State'].unique())))
menu_china = ["China: "+m for m in menu_china]
#menu_us = list(np.sort(list(confirmed[confirmed['Country/Region'] == "US"]['Province/State'].unique())))
#menu_us = ["US: "+m for m in menu_us]
menu_canada = list(np.sort(list(confirmed[confirmed['Country/Region'] == "Canada"]['Province/State'].unique())))
menu_canada = ["Canada: "+m for m in menu_canada]
menu_australia = list(np.sort(list(confirmed[confirmed['Country/Region'] == "Australia"]['Province/State'].unique())))
menu_australia = ["Australia: "+m for m in menu_australia]
dropdown_countries = Select(title = "Country/Region", value = "World", options = menu_countries+menu_china+menu_canada+menu_australia)
dropdown_countries.on_change('value',check_dropdown_update_country)

layout = layout([[div1,plogo],[tabs,],[dropdown_countries,],[table,]])
curdoc().add_root(layout)


from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from django.urls import reverse

from flowers_agent.messenger_bot.helpers import FbMeProfileAPI

class Command(BaseCommand):
    '''
    Reference https://developers.facebook.com/docs/messenger-platform/reference/messenger-profile-api/domain
    -whitelisting
    '''
    help = 'Set messenger whitelist urls'

    def handle(self, *args, **options):
        api = FbMeProfileAPI(getattr(settings, 'MESSENGER_ACCESS_TOKEN'))
        payload = {
            "whitelisted_domains": [
                settings.PUBLIC_URL
            ]
        }
        result = api.post(json=payload)
        if result.get('result') == 'success':
            self.stdout.write(self.style.SUCCESS('Successfully set whitelist domains.'))
        else:
            self.stderr.write(self.style.ERROR('Something went wrong. Details: ${}'.format(result)))


"""Functions for generating synthetic clusters convolved with Gaia measurement errors."""

import numpy as np
import pandas as pd
from astropy.io import ascii
from astropy.coordinates import SkyCoord, CartesianDifferential, CartesianRepresentation
import astropy.units as u
from ..calculate.constants import gaia_dr2_a_lambda_over_a_v, gaia_dr2_zero_points
from ..calculate import king_surface_density, king_surface_density_fast
from scipy.stats import norm, median_absolute_deviation

from pathlib import Path
from typing import Union, Optional, Callable

def _calculate_m_over_h(
    z,
    z_over_x_solar: float = 0.0207,
    y_constant: float = 0.2485,
    z_multiplier: float = 1.78,
):
    """Convenience function for calculating [M/H] given a parsec isochrone.

    See "Ages/metallicities" at: http://stev.oapd.inaf.it/cgi-bin/cmd (last retrieved Feb 2020)

    """
    y = y_constant + z_multiplier * z
    x = 1 - y - z
    return np.log10(z / x) - np.log10(z_over_x_solar)

class SimulatedPopulations:
    def __init__(
        self,
        location: Union[Path, str],
        search_pattern: str = "*.dat",
        mass_tolerance: float = 0.05,
        max_mass_iterations: int = 1000,
        position_oversampling_factor: Union[float, int] = 1.0,
        limiting_magnitude: float = 22.0,
        binary_offset: float = -0.1,
        binary_fraction: float = 0.4,
    ):
        """Convenience class to read in simulated stellar population output from the CMD 3.3 tool into a DataFrame. Also
        stores useful information about the simulated populations (e.g. available unique values) and provides methods to
        access the population.

        Clusters are read in using astropy.io.ascii.

        Note: it is assumed that simulated populations are on a *grid of values*, and this class will fail later if they
        aren't! Aka, there must be the same different ages for every different metallicity.

        Args:
            location (Path, str): location of the single file or directory to cycle over.
            search_pattern (str): if location is a directory, only read in file names that match this pattern. Note that
                * is treated as a wildcard.
                Default: "*.dat" (default extension of CMD 3.3 files)
            mass_tolerance (float): fractional tolerance within which to allow a drawn clusters' mass to deviate from
                the target mass. If set too low, it may not be possible to find a solution for the cluster!
                Default: 0.05
            max_mass_iterations (int): maximum number of times to try and re-draw stars to get a cluster of the correct
                mass before giving up and raising a ValueError.
                Default: 1000
            position_oversampling_factor (float, int): by what factor too many position values do we draw. We calculate
                the expectation value of successes for a given core & tidal radius, then draw this factor too many
                to reduce the number of times we have to call again.
                Set this lower if you're having memory issues!
                Default: 1 (aka about half of the time we'll already have drawn enough random values to move on.)
            limiting_magnitude (float, int): what limiting magnitude we'll immediately drop all stars below. Should be
                set conservatively (and below the actual limiting magnitude) else the far end of clusters will be
                underpopulated, as stars below the limit could scatter back within the bounds of the limit. Set to
                np.inf to turn this off.
                Default: 22., which is very conservative (few stars in Gaia DR2 have photometric errors of more than 0.1
                    even)
            binary_offset (float): mean offset of unresolved binary stars to use. Binary offsets are approximated with
                an exponential distribution of this value.
                Default: -0.1  (from Yen+19 thesis)
            binary_fraction (float): fraction of stars with unresolved binaries.
                Default: 0.4   (typical of many open clusters)

        """
        # Store some class stuff
        self.mass_tolerance = mass_tolerance
        self.max_mass_iterations = max_mass_iterations
        self.required_info_on_clusters = [
            "ra",
            "dec",
            "distance",
            "extinction_v",
            "pmra",
            "pmdec",
            "age",
            "Z",
            "mass",
            "tidal_radius",
            "v_int",
        ]
        self.position_oversampling_factor = position_oversampling_factor
        self.limiting_magnitude = limiting_magnitude
        self.binary_offset = binary_offset
        self.binary_fraction = binary_fraction

        # Cast the path as a pathlib.Path and check that it's real
        location = Path(location)

        if location.exists() is False:
            raise ValueError("specified path does not seem to exist!")

        # Make a list of all files to read in
        if location.is_dir():
            files_to_read = list(location.glob(search_pattern))
        else:
            files_to_read = [location]

        # Cycle over files... reading them in!!!
        separate_dataframes = []
        for a_file in files_to_read:
            separate_dataframes.append(ascii.read(str(a_file.resolve())).to_pandas())

        # Join all the data frames together and be super paranoid about the indexes being correct
        self.data = pd.concat(separate_dataframes, ignore_index=True).reset_index(
            drop=True
        )

        self.data.columns = [
            "Z",
            "age",
            "Mini",
            "Mass",
            "logL",
            "logTe",
            "logg",
            "cmd_label",
            "mbolmag",
            "phot_g_mean_mag",
            "phot_bp_mean_mag",
            "phot_rp_mean_mag",
        ]

        self.data["log_age"] = np.log10(self.data["age"])
        self.data["log_Z"] = _calculate_m_over_h(self.data["Z"])

        # We also want to keep a record of the unique values that we have
        self.unique_log_ages = np.unique(self.data["log_age"].to_numpy())
        self.unique_log_metallicities = np.unique(self.data["log_Z"].to_numpy())

    def _add_positions_to_cluster(
        self, new_population: pd.DataFrame, data_cluster: pd.Series
    ):
        """Gives every star in a cluster a position, randomly assigned by drawing from a King profile in a brute-force
        Monte-Carlo way. Uses self.position_oversampling_factor to control how much we over or under-draw vs. the
        expected number of cluster member stars.

        Args:
            new_population (pd.DataFrame): the population of stars to work on, so far.
            data_cluster (pd.Series): the data for the cluster to work on (see .get_clusters() for full docs), but just
                for the one cluster.

        Returns:
            new_population (pd.DataFrame) but with new photometry appended.

        """
        # Find the maximum value of a King profile given these parameters, making our MC in a moment more efficient
        # This is *NOT* done with the fast version, as the slow version has checks to ensure the values are correct.
        max_king = king_surface_density(
            0, data_cluster["radius_c"], data_cluster["radius_t"]
        )

        # Calculate the expected acceptance fraction with the area under the curve (aka the inv. normalisation constant)
        # (since 1/A = (max) / (A * max) = area under the curve)
        inverse_normalisation_constant = max_king / king_surface_density(
            0, data_cluster["radius_c"], data_cluster["radius_t"], normalise=True
        )

        # The expected acceptance fraction is the ratio of the King curve area to the area of the uniform deviates
        # square, then cubed as we're in 3D.
        expected_acceptance_fraction = (
            inverse_normalisation_constant / (data_cluster["radius_t"] * max_king)
        ) ** 3

        # Grab some stats about how many stars we're working with
        total_stars = new_population.shape[0]
        remaining_stars = total_stars
        completed_stars = 0

        """ OLD WAY - valid only for 1D!
                # Keep an array of final r values to write to and an array of bools for which r values still need to be re-drawn
                # (initialised as an array of only True as we have to begin)
                r_values = np.empty(total_stars, dtype=float)
                to_draw = np.ones(total_stars, dtype=bool)

                # Loop time! Draw random r values and a number between 0,max_king to decide whether or not to accept the r value
                while remaining_stars > 0:
                    # Pull some random deviates from a square
                    r_values[to_draw] = np.random.rand(remaining_stars) * data_cluster['radius_t']
                    test_values = np.random.rand(remaining_stars) * max_king

                    # Evaluate the King profile at these points
                    king_values = king_surface_density(
                        r_values[to_draw], data_cluster['radius_c'], data_cluster['radius_t'])

                    # Tell the loop whether or not to keep drawing these values or whether to accept them
                    to_draw[to_draw] = test_values > king_values

                    remaining_stars = np.count_nonzero(to_draw)

                # Draw random angles for all the stars
                theta, phi = points_on_sphere(total_stars, radians=False, phi_symmetric=True)
                """

        """
        # VALID FOR 3D - but slow as fuck!
        # Keep an array of final r values to write to and an array of bools for which r values still need to be re-drawn
        # (initialised as an array of only True as we have to begin)
        r_values = np.empty((total_stars, 3), dtype=float)
        to_draw = np.ones(total_stars, dtype=bool)

        # Loop time! Draw random r values and a number between 0,max_king to decide whether or not to accept the r value
        while remaining_stars > 0:
            # Pull some random deviates from a square
            r_values[to_draw, :] = (np.random.rand(remaining_stars, 3) * 2 - 1) * data_cluster['radius_t']
            test_values = np.random.rand(remaining_stars, 3) * max_king

            # Evaluate the King profile at these points
            king_values = king_surface_density_fast(
                np.abs(r_values[to_draw, :]), data_cluster['radius_c'], data_cluster['radius_t'])

            # Tell the loop whether or not to keep drawing these values or whether to accept them
            to_draw[to_draw] = np.any(test_values > king_values, axis=1)

            # print(np.any(test_values > king_values, axis=1))

            remaining_stars = np.count_nonzero(to_draw)

        """

        # Keep an array of final r values to write to and an array of bools for which r values still need to be re-drawn
        # (initialised as an array of only True as we have to begin)
        r_values = np.empty((total_stars, 3), dtype=float)

        # Loop time! Draw random r values and a number between 0,max_king to decide whether or not to accept the r value
        while remaining_stars > 0:
            # Pull some random deviates from a square - we pull out a lot more than we need so we can rejection sample
            # in bulk
            stars_to_draw = int(
                remaining_stars
                * self.position_oversampling_factor
                / expected_acceptance_fraction
            )
            test_r_values = (np.random.rand(stars_to_draw, 3) * 2 - 1) * data_cluster[
                "radius_t"
            ]
            test_values = np.random.rand(*test_r_values.shape) * max_king

            # Evaluate the King profile at these points
            king_values = king_surface_density_fast(
                np.abs(test_r_values),
                data_cluster["radius_c"],
                data_cluster["radius_t"],
            )

            # Find the good values
            good_test_values = np.all(test_values < king_values, axis=1)
            n_good_test_values = np.count_nonzero(good_test_values)

            # Clip the number to write, as we don't want to try and write too many!
            if n_good_test_values > remaining_stars:
                values_to_write = remaining_stars
                index_when_we_have_enough_values = (
                    (np.cumsum(good_test_values) == values_to_write).nonzero()[0]
                )[0]
                good_test_values[index_when_we_have_enough_values + 1 :] = False
            else:
                values_to_write = n_good_test_values

            # Write as many values as we can
            r_values[
                completed_stars : completed_stars + values_to_write
            ] = test_r_values[good_test_values]

            # Update with however many stars we wrote
            remaining_stars -= values_to_write
            completed_stars = total_stars - remaining_stars

        # Convert the cluster's stars and the cluster's own position into cartesian coords (we assume that the cluster
        # stars are all around the sun at first, hijacking astropy as a way to go from 3D sphericals to cartesian coords
        cluster_location = SkyCoord(
            ra=data_cluster["ra"] << u.deg,
            dec=data_cluster["dec"] << u.deg,
            distance=data_cluster["distance"] << u.pc,
        ).cartesian

        cluster_stars = CartesianRepresentation(r_values << u.pc, xyz_axis=1)

        cluster_stars = SkyCoord(cluster_location + cluster_stars, frame="icrs")

        # Pop this all back in the DataFrame
        new_population["ra"] = cluster_stars.ra.value
        new_population["dec"] = cluster_stars.dec.value

        # Also make l, b co-ordinates, distances and parallaxes in mas
        cluster_stars = cluster_stars.transform_to("galactic")
        new_population["l"] = cluster_stars.l.value
        new_population["b"] = cluster_stars.b.value
        new_population["distance"] = cluster_stars.distance.value
        new_population["parallax"] = 1000 / new_population["distance"]

        return new_population

    @staticmethod
    def _add_proper_motions_to_cluster(
        new_population: pd.DataFrame, data_cluster: pd.Series
    ):
        """Gives every star in a cluster a proper motion. Requires that they already have positions!

        Args:
            new_population (pd.DataFrame): the population of stars to work on, so far. Must contain 'ra', 'dec' and
                'distance' values!
            data_cluster (pd.Series): the data for the cluster to work on (see .get_clusters() for full docs), but just
                for the one cluster.

        Returns:
            new_population (pd.DataFrame) but with new photometry appended.

        """
        # Convert the cluster location into cartesians.
        cluster_location = SkyCoord(
            ra=data_cluster["ra"] << u.deg,
            dec=data_cluster["dec"] << u.deg,
            distance=data_cluster["distance"] << u.pc,
            pm_ra_cosdec=data_cluster["pmra"] << u.mas / u.yr,
            pm_dec=data_cluster["pmdec"] << u.mas / u.yr,
            radial_velocity=0.0 * u.m / u.s,
            frame="icrs",
        )
        cluster_velocity = cluster_location.velocity

        # Draw Gaussian internal velocities based on the requested internal velocity dispersion.
        v_x, v_y, v_z = norm.rvs(
            loc=0.0, scale=data_cluster["v_int"], size=(3, new_population.shape[0])
        )

        # Add these velocities to the cluster stars' locations to get proper motions
        cluster_stars = SkyCoord(
            ra=new_population["ra"] << u.deg,
            dec=new_population["dec"] << u.deg,
            distance=new_population["distance"] << u.pc,
            frame="icrs",
        ).cartesian

        cluster_star_velocities = CartesianDifferential(
            d_x=(v_x << u.m / u.s) + cluster_velocity.d_x,
            d_y=(v_y << u.m / u.s) + cluster_velocity.d_y,
            d_z=(v_z << u.m / u.s) + cluster_velocity.d_z,
            unit=u.m / u.s,
        )

        cluster_stars = SkyCoord(
            cluster_stars.with_differentials(cluster_star_velocities), frame="icrs"
        )

        # Add the proper motions to the DataFrame
        new_population["pmra"] = cluster_stars.pm_ra_cosdec.value
        new_population["pmdec"] = cluster_stars.pm_dec.value

        return new_population

    def _add_photometry_to_cluster(
        self, new_population: pd.DataFrame, data_cluster: pd.Series
    ):
        """Adds Gaia photometry to a cluster's member stars.

        Args:
            new_population (pd.DataFrame): the population of stars to work on, so far.
            data_cluster (pd.Series): the data for the cluster to work on (see .get_clusters() for full docs), but just
                for the one cluster.

        Returns:
            new_population (pd.DataFrame) but with new photometry appended.

        """
        # Make some binary offsets (just in the first bit of the array, but then shuffled)
        n_stars = new_population.shape[0]
        n_binaries = int(self.binary_fraction * n_stars)
        binary_offsets = np.zeros(n_stars)
        binary_offsets[:n_binaries] = np.sign(
            self.binary_offset
        ) * np.random.exponential(np.abs(self.binary_offset), size=n_binaries)
        binary_offsets = np.random.permutation(binary_offsets)

        # Make the magnitudes
        new_population["phot_g_mean_mag"] += (
            data_cluster["distance_modulus"]
            + data_cluster["extinction_v"] * gaia_dr2_a_lambda_over_a_v["G"]
            + binary_offsets
        )
        new_population["phot_bp_mean_mag"] += (
            data_cluster["distance_modulus"]
            + data_cluster["extinction_v"] * gaia_dr2_a_lambda_over_a_v["G_BP"]
        )
        new_population["phot_rp_mean_mag"] += (
            data_cluster["distance_modulus"]
            + data_cluster["extinction_v"] * gaia_dr2_a_lambda_over_a_v["G_RP"]
        )
        new_population["bp_rp"] = (
            new_population["phot_bp_mean_mag"] - new_population["phot_rp_mean_mag"]
        )

        # Reverse-engineer some fluxes
        new_population["phot_g_mean_flux"] = 10 ** (
            (gaia_dr2_zero_points["G"] - new_population["phot_g_mean_mag"]) / 2.5
        )
        new_population["phot_bp_mean_flux"] = 10 ** (
            (gaia_dr2_zero_points["G_BP"] - new_population["phot_bp_mean_mag"]) / 2.5
        )
        new_population["phot_rp_mean_flux"] = 10 ** (
            (gaia_dr2_zero_points["G_RP"] - new_population["phot_rp_mean_mag"]) / 2.5
        )

        return new_population

    def _draw_cluster_of_correct_mass(
        self, simulated_population: pd.DataFrame, target_mass: float
    ):
        """Draws random samples from a simulated population and creates a cluster of the desired mass. Uses
        self.mass_tolerance and self.max_mass_iterations to govern the random sampling. These can be set during class
        initialisation.

        Args:
            simulated_population (pd.DataFrame): a simulated stellar population at the required Z, age.
            target_mass (float): the final mass to aim for when generating a cluster.

        Returns:
            new_population (pd.DataFrame): a data frame of all the details of the newly generated cluster.

        """
        # Some stuff we'll need
        n_simulated_stars = simulated_population.shape[0]
        total_simulated_mass = np.sum(simulated_population["Mass"])
        typical_mass = np.mean(simulated_population["Mass"])

        # First, we try to get close by randomly selecting stars in the simulated population
        fraction_of_stars_to_draw = target_mass / total_simulated_mass
        stars_to_draw = int(target_mass / total_simulated_mass * n_simulated_stars)
        new_indices = np.random.randint(0, high=n_simulated_stars, size=stars_to_draw)

        # Now, we'll try to re-sample the cluster until the new mass is within the mass_tolerance of the target mass
        new_mass = np.sum(simulated_population.loc[new_indices, "Mass"])
        mass_difference = target_mass - new_mass
        iterations = 0
        while np.abs(mass_difference) > target_mass * self.mass_tolerance:
            # Decide whether to add a star or take one away
            n_stars_to_draw = int(np.clip(mass_difference / typical_mass, 1, np.inf))
            if mass_difference > 0:
                new_indices = np.append(
                    new_indices,
                    np.random.randint(0, high=n_simulated_stars, size=n_stars_to_draw),
                )
            else:
                new_indices = np.delete(
                    new_indices,
                    np.random.randint(
                        0, high=new_indices.shape[0], size=n_stars_to_draw
                    ),
                )

            new_mass = np.sum(simulated_population.loc[new_indices, "Mass"])
            mass_difference = target_mass - new_mass

            iterations += 1
            if iterations > self.max_mass_iterations:
                raise ValueError(
                    f"unable to select a cluster of the correct mass after {iterations} iterations! Try "
                    f"reducing the tolerance of the desired cluster mass."
                )

        return simulated_population.loc[new_indices, :].reset_index(drop=True)

    def _make_cluster(self, data_cluster: pd.Series, cluster_label: int):
        """The actual function in the class that iteratively makes clusters one by one! Calls all the other methods
        around these parts and returns a whole cluster once done.

        Args:
            data_cluster (pd.Series): the data for the cluster to work on (see .get_clusters() for full docs), but just
                for the one cluster.
            cluster_label (int): the label to append to cluster stars. Useful to keep track of which star belongs
                where if the overall output of .get_clusters() is concatenated later.

        Returns:
            new_population (pd.DataFrame): a data frame of all the details of the newly generated cluster.

        """
        # Grab the stars that be good from the simulated populations
        good_stars = np.logical_and(
            self.data["log_age"] == data_cluster["age"],
            self.data["log_Z"] == data_cluster["Z"],
        )

        # Check that something hasn't gone wrong and that at least one star matches this combination
        if np.count_nonzero(good_stars) == 0:
            raise ValueError(
                f"specified age {data_cluster['age']} and metallicity {data_cluster['Z']} combination "
                f"was not found in the simulated population!"
            )

        # Turn this into a new DataFrame! (I don't want to mess up the old one)
        simulated_population = (
            self.data.loc[good_stars, :].copy().reset_index(drop=True)
        )

        # Shuffle the DataFrame and create a stellar sample of roughly the required mass
        new_population = self._draw_cluster_of_correct_mass(
            simulated_population, data_cluster["mass"]
        )

        # Add parameters to the cluster & perform dropping if requested
        new_population = self._add_photometry_to_cluster(new_population, data_cluster)
        new_population = new_population.loc[
            new_population["phot_g_mean_mag"] < self.limiting_magnitude, :
        ].reset_index(drop=True)
        new_population = self._add_positions_to_cluster(new_population, data_cluster)
        new_population = self._add_proper_motions_to_cluster(
            new_population, data_cluster
        )
        new_population["cluster_label"] = cluster_label

        return new_population

    def get_clusters(
        self,
        data_clusters: pd.DataFrame,
        error_on_invalid_request: bool = True,
        concatenate: bool = True,
    ):
        """Creates random samples of stars given an input log_ages and log_metallicities.

        Warning: will use the closest available population, which means you ought to check that the input age and
        metallicity given are close to what was given when the class was made!

        Args:
            data_clusters (pd.DataFrame): information on all clusters to return, including:
                Spatial parameters
                    'ra': right ascension of the cluster. (deg)
                    'dec': declination of the cluster. (deg)
                    'distance': distance to the cluster. (parsecs)
                    'extinction_v': v-band extinction towards the cluster.
                    'pmra': proper motion in right ascension of the cluster. (mas/yr)
                    'pmdec': proper motion in declination of the cluster. (mas/yr)
                Internal parameters
                    'age': logarithmic age of the cluster.
                    'Z': metallicity of the cluster, in units of dex (aka [Fe/H] or Z/X).
                    'mass': underlying mass of the cluster, in solar masses.
                    'radius_c': King's core radius of the cluster, in parsecs.
                    'radius_t': King's tidal radius of the cluster, in parsecs.
                    'v_int': internal velocity dispersion of the cluster, in m/s.
            error_on_invalid_request (bool): whether or not to raise an error if the requested age and metallicity
                values are not within the range of the catalogue.
                Default: True
            concatenate (bool): whether or not to join all the clusters together into one DataFrame.
                Default: True

        Returns:
            a single pd.DataFrame or a list of pd.DataFrame

        """
        # Get the nearest ages and metallicities of the required stars vs the input stars
        age_differences = np.abs(
            np.asarray(data_clusters["age"]).reshape(-1, 1) - self.unique_log_ages
        )
        metallicity_differences = np.abs(
            np.asarray(data_clusters["Z"]).reshape(-1, 1)
            - self.unique_log_metallicities
        )
        indices_best_age = np.argmin(age_differences, axis=1)
        indices_best_metallicity = np.argmin(metallicity_differences, axis=1)
        n_stars = data_clusters.shape[0]

        # Check the user isn't being a numpty
        if error_on_invalid_request:
            if np.any(
                np.logical_or(
                    age_differences[np.arange(n_stars), indices_best_age] > 1,
                    metallicity_differences[
                        np.arange(n_stars), indices_best_metallicity
                    ]
                    > 1,
                )
            ):
                raise ValueError(
                    "an input age or metallicity value is too different (more than 1) from the values \n"
                    "available in the catalogue! Set error_on_invalid_request=False to disable this \n"
                    "warning, albeit at your own risk as output clusters will not match your \n"
                    "specifications in age or metallicity."
                )

        # Grab the best values for each based on what's near to the simulated population
        data_clusters["age"] = self.unique_log_ages[indices_best_age]
        data_clusters["Z"] = self.unique_log_metallicities[indices_best_metallicity]

        # Grab some stuff we'll need in a moment: distance modulus and limiting magnitude
        data_clusters["distance_modulus"] = 5 * np.log10(data_clusters["distance"]) - 5

        # Loop over all of the requested ages and metallicities, and make clusters of about the same mass
        n_clusters = data_clusters.shape[0]
        clusters_to_return = []
        i_cluster = 0
        while i_cluster < n_clusters:
            clusters_to_return.append(
                self._make_cluster(data_clusters.loc[i_cluster, :], i_cluster)
            )
            i_cluster += 1

        # Concatenate if requested and return
        if concatenate:
            return pd.concat(clusters_to_return, ignore_index=True).reset_index(
                drop=True
            )
        else:
            return clusters_to_return

def _c_position_limits_plus_minus_two(data, size, name):
    return np.asarray([np.min(data[name]) - 2, np.max(data[name]) + 2])

def _c_random_value(data, size, name):
    return np.random.choice(data[name], size=size)

def _c_random_cbj_distance(data, size, name):
    return np.random.choice(data["r_est"], size=size)

def _c_median_plus_or_minus_1_sigma(data, size, name):
    return np.median(data[name]) * np.asarray([+1, -1]) * np.std(data[name])

def _c_median_plus_or_minus_1_mad(data, size, name):
    return (
        np.median(data[name])
        * np.asarray([+1, -1])
        * median_absolute_deviation(data[name])
    )

def _c_median(data, size, name):
    return np.repeat(np.median(data[name]), size)

def _c_median_distance(data, size, name):
    return np.repeat(np.median(1000 / data["parallax"]), size)

_valid_modes = ["clustering_augmentation", "generator"]

_default_cluster_parameters_augmentation = {
    # Spatial parameters
    "ra": _c_position_limits_plus_minus_two,
    "dec": _c_position_limits_plus_minus_two,
    "distance": _c_median_distance,
    "extinction_v": 0.0,
    "pmra": _c_median_plus_or_minus_1_mad,
    "pmdec": _c_median_plus_or_minus_1_mad,
    # Internal parameters
    "age": 7.0,
    "Z": 0.0,
    "mass": 1e3,
    "radius_c": 1.5,
    "radius_t": 10.0,
    "v_int": 500.0,
}

_default_cluster_parameters_generator = {
    # Spatial parameters
    "ra": _c_random_value,
    "dec": _c_random_value,
    "distance": _c_median_distance,
    "extinction_v": 0.0,
    "pmra": _c_random_value,
    "pmdec": _c_random_value,
    # Internal parameters
    "age": 7.0,
    "Z": 0.0,
    "mass": 1e3,
    "radius_c": 1.5,
    "radius_t": 10.0,
    "v_int": 500.0,
}

def _setup_cluster_parameter(
    data_gaia: pd.DataFrame,
    parameter_name: str,
    desired_parameter_values: Union[np.ndarray, float, int, Callable],
    n_clusters: int,
):
    """Sets up information for data_cluster (the dataframe of all clusters to make) based on given information, one
    parameter at a time.

    Args:
        data_gaia (pd.DataFrame): Gaia data that may be used by a callable arg.
        parameter_name (str): name (as with standard Gaia names) of the parameter to call. E.g. 'ra'.
        desired_parameter_values (np.ndarray, float, int, callable): what to set the parameter values to. Type dictates
            what happens:
            float/int: set to be the same for all clusters
            np.ndarray: values are randomly sampled from this array.
            callable: values are returned by this function. Function should accept three args:
                - the dataframe data_gaia
                - the number of clusters
                - the name of the parameter.
        n_clusters (int): number of clusters requiring parameters.

    Returns:
        a np.ndarray of cluster parameters.

    """
    # If desired_parameter_values is a np.ndarray, then we assume that it's something for us to pull values from
    # randomly
    if isinstance(desired_parameter_values, np.ndarray):
        return np.random.choice(desired_parameter_values, size=n_clusters)

    # Otherwise, if it's a float or an int, then we simply want to copy that value across
    elif isinstance(desired_parameter_values, (float, int)):
        return np.ones(n_clusters, dtype=float) * desired_parameter_values

    # Otherwise, if it's a function, then we want
    elif callable(desired_parameter_values):
        return desired_parameter_values(data_gaia, n_clusters, parameter_name)

    else:
        raise ValueError(
            "specified desired_parameter_values has an unsupported type. "
            "It should be an int, float, np.ndarray or callable."
        )

parameters_with_symmetric_error = [
    "pmra",
    "pmdec",
    "parallax",
    "phot_g_mean_flux",
    "phot_bp_mean_flux",
    "phot_rp_mean_flux",
]

magnitude_parameters = ["phot_g_mean_mag", "phot_bp_mean_mag", "phot_rp_mean_mag"]

magnitude_parameter_band_names = ["G", "G_BP", "G_RP"]

bp_rp_color_name = "bp_rp"

flux_parameters = ["phot_g_mean_flux", "phot_bp_mean_flux", "phot_rp_mean_flux"]

parameters_to_assign = ["ruwe"]

distance_parameters_with_assymetric_error = ["distance"]

# Assymetric keys should live in a dictionary (whose keys are the same as distance_parameters_with_assymetric_error)
# and each entry should look have [modal parameter, low parameter, high parameter, original parameter] keys in a list.
distance_parameters_with_assymetric_error_keys = {
    "distance": ["r_est", "r_lo", "r_hi", "parallax"]
}

def _find_nearest_magnitude_star(
    gaia_photometry: Union[pd.Series, np.ndarray],
    simulated_photometry: Union[pd.Series, np.ndarray],
    fiddle_factor: float = 0.01,
):
    """Little function to do some array maths and find the nearest star to a synthetic one. Includes a fiddle factor
    to make sure that if two identical stars are picked from simulated data, they're less likely to end up with the same
    errors.

    Args:
        gaia_photometry (pd.Series, np.ndarray): gaia photometry to find matches in.
        simulated_photometry (pd.Series, np.ndarray): synthetic photometry to find matches for.
        fiddle_factor (float): standard deviation of normal deviates to add to synthetic photometry to help prevent
            identical errors across the board.
            Default: 0.01

    Returns:
        np.ndarray of the indices into gaia_photometry that returned the closest matches for the simulated_photometry.

    """
    # First, we add the fiddle factor onto the simulated stars so that if two identical stars are picked from simulated
    # data, they're less likely to end up with the same errors
    simulated_photometry += np.random.normal(
        loc=0.0, scale=fiddle_factor, size=simulated_photometry.shape
    )

    # Then, we find the nearest magnitude star (in the G band) to each simulated star (brute force because
    # Emily is Lazy TM) and return this
    return np.argmin(
        np.abs(
            np.asarray(gaia_photometry).reshape(1, -1)
            - np.asarray(simulated_photometry).reshape(-1, 1)
        ),
        axis=1,
    )

def _convolve_simulated_clusters_with_errors(
    data_gaia: pd.DataFrame, data_simulated: pd.DataFrame
):
    """Adds errors onto synthetic photometry. Method for assymetric errors is currently "quite shit."

    Notes:
        See parameters_with_symmetric_error and
        distance_parameters_with_assymetric_error/distance_parameters_with_assymetric_error_keys to see lists of params
        that will be added by this function to data_simulated. These can be changed after import of the package.

    Args:
        data_gaia (pd.DataFrame): gaia data of shape data_simulated.shape, where each entry is the nearest found entry
            to data_simulated. INDEX MUST BE RESET BEFORE CALLING.
        data_simulated (pd.DataFrame): simulated data to add errors to.

    Returns:
        data_simulated, but with errors added!

    """

    # Idiot check
    n_stars = data_simulated.shape[0]
    if data_gaia.shape[0] != n_stars:
        raise ValueError(
            "data_gaia must already have been pre-selected to be the best matches to data_simulated, and "
            "hence both data frames must have the same shape. But they do not!! Shape mismatch is "
            f"{data_gaia.shape} for Gaia vs. {data_simulated.shape} for simulated data"
        )

    # Deal with assymmetric parameters, which are also allowed to have weird keys (this was basically just made to deal
    # with all the different distance estimators that exist in the modern world today)
    # Todo this could infer errors in the actual way used (e.g. with CBJ distances) as a way to be less shit
    for a_parameter in distance_parameters_with_assymetric_error:
        (
            a_parameter_mode,
            a_parameter_low,
            a_parameter_high,
            original_parameter,
        ) = distance_parameters_with_assymetric_error_keys[a_parameter]

        # Work out what fractional error the matched stars have on their assymetric errors.
        # N.B. low fractional errors have negative signs, high errors have positive signs.
        gaia_fractional_errors = np.asarray(
            (
                data_gaia[[a_parameter_low, a_parameter_high]].to_numpy().reshape(-1, 2)
                - data_gaia[a_parameter_mode].to_numpy().reshape(-1, 1)
            )
            / data_gaia[a_parameter_mode].to_numpy().reshape(-1, 1)
        )

        # Do some TOTAL BULLSHIT to try and make a rough estimate of what the errors should be.
        # We pull a random value in the range of the fractional errors (loosely equivalent to +- 1 sigma, which might be
        # too small but I don't want to let negative values happen)
        use_low_or_high = np.random.choice([0, 1], size=n_stars)

        data_simulated[a_parameter_mode] = data_simulated["distance"] * (
            1
            + np.random.randn(n_stars)
            * gaia_fractional_errors[np.arange(n_stars), use_low_or_high]
        )

        data_simulated[a_parameter_low] = data_simulated[
            a_parameter_mode
        ].to_numpy() * (1 + gaia_fractional_errors[:, 0])
        data_simulated[a_parameter_high] = data_simulated[
            a_parameter_mode
        ].to_numpy() * (1 + gaia_fractional_errors[:, 1])

    # Deal with symmetric parameters, adding some lovely symmetric error bars to the data
    for a_parameter in parameters_with_symmetric_error:
        a_parameter_error = a_parameter + "_error"
        data_simulated[a_parameter_error] = data_gaia[a_parameter_error]
        data_simulated[a_parameter] += np.random.normal(
            0.0, scale=data_simulated[a_parameter_error], size=n_stars
        )

    # Deal with magnitudes, which we'll want to re-calculate now that the photometric information has changed
    for a_magnitude_parameter, a_band, a_flux in zip(
        magnitude_parameters, magnitude_parameter_band_names, flux_parameters
    ):
        data_simulated[a_magnitude_parameter] = (
            -2.5 * np.log10(data_simulated[a_flux]) + gaia_dr2_zero_points[a_band]
        )

    # Add a new bp-rp colour
    if bp_rp_color_name is not None:
        data_simulated[bp_rp_color_name] = (
            data_simulated["phot_bp_mean_mag"] - data_simulated["phot_rp_mean_mag"]
        )

    # Finally, deal with copy-pasted parameters
    for a_parameter_to_assign in parameters_to_assign:
        data_simulated[a_parameter_to_assign] = data_gaia[a_parameter_to_assign]

    return data_simulated

def generate_synthetic_clusters(
    simulated_populations: SimulatedPopulations,
    data_gaia: pd.DataFrame,
    mode: str = "clustering_augmentation",
    cluster_parameters_to_overwrite: Optional[dict] = None,
    kwargs_for_simulated_populations: Optional[dict] = None,
    n_clusters: int = 2,
    concatenate: bool = True,
    shuffle: bool = True,
):
    """Generates synthetic star clusters based on generated simulated populations (e.g. via CMD 3.3) and real examples
    of Gaia data (and its associated errors.)

    Args:
        simulated_populations (SimulatedPopulations object): an ocelot.cluster.SimulatedPopulations object with data
            on simulated populations pre-loaded.
        data_gaia (pd.DataFrame): a representative sample of Gaia stars to model errors with.
        mode (str): mode to use for generation. Accepts the following arguments:
            'clustering_augmentation': will generate two clusters outside of the central field and at the median
                distance. This setting is designed to be used with algorithms like HDBSCAN to make tree splitting more
                sensible. Every entry in cluster_parameters must have a max length of two.
            'generator': will generate as many clusters as you'd like. Ideal for when a large sample of simulated OCs
                is needed.
        cluster_parameters_to_overwrite (dict, optional): parameters for the clusters to generate. The following have
            default values and can be changed:
                Spatial parameters
                    'ra': right ascension of the cluster. (deg) default: +- min and max
                    'dec': declination of the cluster. (deg) default: inferred
                    'distance': distance to the cluster. (parsecs) default: median of data_gaia
                    'extinction_v': v-band extinction towards the cluster. default: 0.0
                    'pmra': proper motion in right ascension of the cluster. (mas/yr) default: +-4 sigma of data_gaia
                    'pmdec': proper motion in declination of the cluster. (mas/yr) default: +-4 sigma of data_gaia
                Internal parameters
                    'age': logarithmic age of the cluster. default: 7.0
                    'Z': metallicity of the cluster, in units of dex (aka [Fe/H] or Z/X). default: 0.0 (solar)
                    'mass': underlying mass of the cluster, in solar masses. default: 1000
                    'radius_c': King's core radius of the cluster, in parsecs. default: 1.5
                    'radius_t': King's tidal radius of the cluster, in parsecs. default: 10
                    'v_int': internal velocity dispersion of the cluster, in m/s. default: 500
            and each parameter may be a number, an array of values to sample from, or a callable function to apply. The
            callable function must take arguments data (a pd.DataFrame), size (aka the number of clusters)
            and the name of the parameter (a string), and return a np.ndarray or pd.Series of shape (size,).
            Default value of cluster_parameters: None
                (hence uses all defaults, the above are for mode 'clustering_augmentation')
        n_clusters (int): in mode "generator", this specifies the number of clusters to make. In mode
            'clustering_augmentation', this will always be two.
            Default: 2
        concatenate (bool): whether or not to return a concatenated DataFrame of data_gaia and data_simulated, or
            whether to only return data_simulated. In the former case, all stars in data_gaia will also be labelled -1
            in a new cluster_label column to show that they are noise.
            Default: True
        shuffle (bool): whether or not to shuffle the output to hide the simulated clusters in the data.
            Default: True
        kwargs_for_simulated_populations (dict, optional): keyword arguments to pass to the SimulatedPopulations object.
            Default: None

    """
    # Pre-process the easy function arguments
    if mode not in _valid_modes:
        raise ValueError(
            f"specified mode {mode} not supported! You may choose from the following values: \n"
            f"{_valid_modes}"
        )

    if kwargs_for_simulated_populations is None:
        kwargs_for_simulated_populations = {}
    if cluster_parameters_to_overwrite is None:
        cluster_parameters_to_overwrite = {}

    # More setup, but dependant on the mode
    if mode is "clustering_augmentation":
        cluster_parameters = _default_cluster_parameters_augmentation
        n_clusters = 2
    else:  # mode is 'generator'
        cluster_parameters = _default_cluster_parameters_generator

    # Cycle over overwrite parameters
    for a_key in cluster_parameters_to_overwrite.keys():
        cluster_parameters[a_key] = cluster_parameters_to_overwrite[a_key]

    # Process all of the parameters
    data_cluster = pd.DataFrame({})
    for a_key in cluster_parameters.keys():
        data_cluster[a_key] = _setup_cluster_parameter(
            data_gaia, a_key, cluster_parameters[a_key], n_clusters
        )

    # Cluster simulation time!!!!
    data_simulated = simulated_populations.get_clusters(
        data_cluster, **kwargs_for_simulated_populations
    )

    # Convolve some cheeky errors in there too
    star_matches = _find_nearest_magnitude_star(
        data_gaia["phot_g_mean_mag"], data_simulated["phot_g_mean_mag"]
    )
    data_simulated = _convolve_simulated_clusters_with_errors(
        data_gaia.loc[star_matches, :].reset_index(drop=True), data_simulated
    )

    if concatenate:
        # Append a noise cluster label to the original data too
        data_gaia["cluster_label"] = -1
        data_simulated = pd.concat([data_simulated, data_gaia]).reset_index(drop=True)

    # Return an array
    if shuffle:
        return data_simulated.sample(frac=1).reset_index(drop=True)
    else:
        return data_simulated