from __future__ import annotations

import filecmp
import json
import subprocess
from pathlib import Path

from d2b.d2b import __version__

class TestE2e:
    def _check_run_results(
        self,
        data_dir: Path,
        out_dir: Path,
        sidecar_files: list[str],
        other_files: list[str],
    ):
        """generic checks for all test_run_* methods"""
        expected_out_dir = data_dir / "out"

        for f in sidecar_files:
            assert (out_dir / f).exists()

            actual_sidecar = json.loads((out_dir / f).read_text())
            expected_sidecar = json.loads((expected_out_dir / f).read_text())
            assert actual_sidecar.pop("D2bVersion") == __version__
            assert actual_sidecar == expected_sidecar

        for f in other_files:
            assert (out_dir / f).exists()
            assert filecmp.cmp(out_dir / f, expected_out_dir / f, shallow=False)

    def test_cli_run(self, d2b_run_e2e: Path, tmpdir: str):

        data_dir = d2b_run_e2e / "intended-for-fields"
        sidecar_files = [
            "sub-a/ses-1/fmap/sub-a_ses-1_dir-AP_fmap.json",
            "sub-a/ses-1/fmap/sub-a_ses-1_dir-PA_fmap.json",
            "sub-a/ses-1/fmap/sub-a_ses-1_fmap.json",
            "sub-a/ses-1/func/sub-a_ses-1_task-rest_bold.json",
            "sub-a/ses-1/anat/sub-a_ses-1_T1w.json",
            "sub-a/ses-1/func/sub-a_ses-1_task-fingertap_bold.json",
        ]
        other_files = [
            "sub-a/ses-1/fmap/sub-a_ses-1_dir-AP_fmap.nii.gz",
            "sub-a/ses-1/fmap/sub-a_ses-1_dir-PA_fmap.nii.gz",
            "sub-a/ses-1/fmap/sub-a_ses-1_fmap.nii.gz",
            "sub-a/ses-1/func/sub-a_ses-1_task-rest_bold.nii.gz",
            "sub-a/ses-1/anat/sub-a_ses-1_T1w.nii.gz",
            "sub-a/ses-1/func/sub-a_ses-1_task-fingertap_bold.nii.gz",
        ]
        out_dir = Path(tmpdir) / "bids"

        config_file = data_dir / "d2b-config.json"
        in_dir = data_dir / "in"

        subprocess.run(
            (
                "d2b",
                "run",
                f"--config={config_file}",
                "--participant=a",
                "--session=1",
                f"--out-dir={out_dir}",
                in_dir,
            ),
        )

        self._check_run_results(data_dir, out_dir, sidecar_files, other_files)

    def test_cli_scaffold(self, tmpdir: str):
        scaffold_files = ["CHANGES", "README", "dataset_description.json"]
        scaffold_directories = ["code", "derivatives"]

        subprocess.run(("d2b", "scaffold", tmpdir))

        for f in scaffold_files:
            assert (Path(tmpdir) / f).is_file()

        for d in scaffold_directories:
            assert (Path(tmpdir) / d).is_dir()

    def test_cli_scaffold_with_participants_table(self, tmpdir: str):
        scaffold_files = [
            "CHANGES",
            "README",
            "dataset_description.json",
            "participants.json",
            "participants.tsv",
        ]
        scaffold_directories = ["code", "derivatives"]

        subprocess.run(("d2b", "scaffold", "--with-participants-table", tmpdir))

        for f in scaffold_files:
            assert (Path(tmpdir) / f).is_file()

        for d in scaffold_directories:
            assert (Path(tmpdir) / d).is_dir()


#!/usr/bin/python
#Dragonborn

import random

# Driver Function -- Determines Draconic Ancestry
# Also determines Breath Weapon
def Skyrim(r):
	while True:
		if r:
			sel = random.randint(1, 10)
			if sel == 1: Dragon = "black"
			elif sel == 2: Dragon = "blue"
			elif sel == 3: Dragon = "brass"
			elif sel == 4: Dragon = "bronze"
			elif sel == 5: Dragon = "copper"
			elif sel == 6: Dragon = "gold"
			elif sel == 7: Dragon = "green"
			elif sel == 8: Dragon = "red"
			elif sel == 9: Dragon = "silver"
			elif sel == 10: Dragon = "white"
		else:
			print("A number of your racial abilities are tied to your draconic ancestry.")
			print("What kind of dragon does your character descend from?")
			print("Please indicate by color:")
			print("Dragon\tDamage Type\tBreath Weapon")

			print("Black\tAcid\t5' by 30' line (Dex save)")
			print("Blue\tLightning\t5' by 30' line (Dex save)")
			print("Brass\tFire\t5' by 30' line (Dex save)")
			print("Bronze\tLightning\t5' by 30' line (Dex save)")
			print("Copper\tAcid\t5' by 30' line (Dex save)")
			print("Gold\tFire\t15' cone (Dex save)")
			print("Green\tPoison\t15' cone (Con save)")
			print("Red\tFire\t15' cone (Dex save)")
			print("Silver\tCold\t15' cone (Con save)")
			print("White\tCold\t15' cone (Con save)")

			Dragon = input()

		if Dragon.lower() == "black" or Dragon.lower() == "copper":
			Dam_Type = "Acid"
			Breath = "5' by 30' line (Dex save)"
			break
		elif Dragon.lower() == "blue" or Dragon.lower() == "bronze":
			Dam_Type = "Lightning"
			Breath = "5' by 30' line (Dex save)"
			break
		elif Dragon.lower() == "brass":
			Dam_Type = "Fire"
			Breath = "5' by 30' line (Dex save)"
			break
		elif Dragon.lower() == "gold" or Dragon.lower() == "red":
			Dam_Type = "Fire"
			Breath = "15' cone (Dex save)"
			break
		elif Dragon.lower() == "green":
			Dam_Type = "Poison"
			Breath = "15' cone (Con save)"
			break
		elif Dragon.lower() == "silver" or Dragon.lower() == "white":
			Dam_Type = "Cold"
			Breath = "15' cone (Con save)"
			break
		else:
			print("Invalid dragon color")
	return Dragon.lower(), Dam_Type, Breath

## Feature Functions ##
def Ancestry(Dragon):
	Name = "Draconic Ancestry"
	Feature = "You are distantly related to " + Dragon + " dragons, which impacts your other racial features."
	return Name, Feature

def BreathWeapon(Dam_Type, Breath):
	Name = "Breath Weapon"
	Feature = "You can use your action to exhale destructive energy. It deals damage in an area according to your ancestry. When you use your breath weapon, all creatures in the area must make a saving throw, the type of which is determined by your ancestry. The DC of this saving throw is 8 + your Constitution modifier + your proficiency bonus. A creature takes 2d6 damage on a failed save, and half as much damage on a successful one. The damage increase to 3d6 at 6th level, 4d6 at 11th, and 5d6 at 16th level. After using your breath weapon, you cannot use it again until you complete a short or long rest.\nDamage Type = " + Dam_Type + "\nBreath = " + Breath
	return Name, Feature

def Resistance(Dam_Type):
	Name = "Resistance"
	Feature = "You have resistance to " + Dam_Type + " damage."
	return Name, Feature


import gym
from gym.vector import SyncVectorEnv, AsyncVectorEnv
import numpy as np
import time
from axel.a2c.a2c import A2C
from axel.impala.impala import Impala
from axel.ppo.ppo import Ppo
from axel.pop3d.pop3d import Pop3d
from axel.ppg.ppg import Ppg
from axel.common.env_wrappers import apply_wrappers,apply_wrappers_2
from axel.ppo.recurrent_ppo import RecurrentPPO
from axel.trainer_builder import TrainerBuilder
from axel.dddq.dddq import DDDQ

def get_env():
    # return apply_wrappers(env=gym.make(id="ALE/Riverraid-v5"))
    return apply_wrappers(env=gym.make("CarRacing-v2", continuous=False))

def get_env_fn(env_name:str,skip=4,grayscale=True,resize=84,framestack=1,reward_scale=1/100):
    def custom_env():
        return apply_wrappers(env=gym.make(id=env_name),skip=skip,grayscale=grayscale,resize=resize,framestack=framestack,reward_scale=reward_scale)
    return custom_env

def get_cartpole():
    return gym.make("CartPole-v1")

def get_acrobat():
    return gym.make("Acrobot-v1")

def train_pop3d():
    env_fns = [get_env for _ in range(8)]
    vec_env = AsyncVectorEnv(env_fns=env_fns)
    pop3d = Pop3d(vec_env=vec_env, step_size=128,
                  total_steps=int(1e6), beta=10, normalize_rewards=True)
    pop3d.run()
    vec_env.close()

def train_ppo():
    env_fns = [get_env for _ in range(8)]
    vec_env = AsyncVectorEnv(env_fns=env_fns)
    ppo = (TrainerBuilder
            .use_ppo()
            .total_steps(1e6)
            .policy_clip(0.1)
            .enable_rewards_normalisation()
            .disable_advantages_normalisation()
            .vec_env(vec_env)
            .build())
    ppo.run()
    vec_env.close()

def train_ppg():
    env_fns = [get_env for _ in range(8)]
    vec_env = AsyncVectorEnv(env_fns=env_fns)
    ppg = (TrainerBuilder.use_ppg()
                .step_size(256)
                .clip_ratio(0.2)
                .n_pi(16)
                .n_aux_epochs(6)
                .n_pi_epochs(1)
                .n_v_epochs(1)
                .n_batches(8)
                .n_aux_batches(16)
                .learning_rate(5e-4)
                .enable_reward_normalization(3)
                .gamma(0.99)
                .disable_lr_decay()
                .vec_env(vec_env)
                .disable_advantage_normalization()
                .entropy_coef(0)
                .build())
    ppg.run()
    vec_env.close()

def train_a2c():
    env_fns = [get_env for _ in range(8)]
    a2c = A2C(env_fns)
    a2c.run()

def train_impala():
    env_fns = [get_env for _ in range(8)]
    impala = Impala(env_fns)
    impala.run()
def train_ppg_cartpole():
    env_fns = [get_cartpole for _ in range(8)]
    vec_env = AsyncVectorEnv(env_fns=env_fns)
    ppg = Ppg(
        vec_env=vec_env, total_steps=int(1e6),
        step_size=20,
        clip_ratio=0.2,
        n_pi=32,
        n_aux_epochs=6,
        n_pi_epochs=1, n_v_epochs=1,
        n_batches=4,
        n_aux_batches=8,
        lr=3e-4,
        normalize_adv=False,
        entropy_coef=0.00,
        normalize_rewards=True,
        max_grad_norm=1)
    ppg.run()

def train_ppg_acrobat():
    env_fns = [get_acrobat for _ in range(8)]
    vec_env = AsyncVectorEnv(env_fns=env_fns)
    ppg = Ppg(
        vec_env=vec_env, total_steps=int(1e6),
        step_size=20,
        clip_ratio=0.2,
        n_pi=32,
        n_aux_epochs=6,
        n_pi_epochs=1, n_v_epochs=1,
        n_batches=4,
        n_aux_batches=8,
        lr=3e-4,
        normalize_adv=True,
        entropy_coef=0.00,
        normalize_rewards=True,
        max_grad_norm=1)
    ppg.run()

def train_ppo_catrpole():
    env_fns = [get_cartpole for _ in range(8)]
    vec_env = AsyncVectorEnv(env_fns=env_fns)
    # ppo = Ppo(vec_env=vec_env,
    #           step_size=20,
    #           total_steps=int(2e6),
    #           policy_clip=0.1,
    #           entropy_coef=0,
    #           normalize_rewards=False,
    #           normalize_adv=True,
    #           )
    ppo = (TrainerBuilder.use_ppo()
                .use_defaults()
                .vec_env(vec_env)
                .step_size(20)
                .total_steps(2e6)
                .policy_clip(0.1)
                .entropy_coef(0)
                .disable_rewards_normalisation()
                .enable_advantages_normalisation()
                .build())
    ppo.run()

def train_pop3d_cartpole():
    env_fns = [get_cartpole for _ in range(16)]
    vec_env = SyncVectorEnv(env_fns=env_fns)
    pop3d = Pop3d(vec_env,
            total_steps=1e6,
            step_size=20,
            entropy_coef=0,
            beta=10,
            normalize_adv=True
            )
    pop3d.run()

def train_recurrent_ppo():
    envs_fns = [get_env_fn("ALE/Riverraid-v5",reward_scale=1/100,framestack=1) for _ in range(8)]
    ppo = RecurrentPPO(total_steps=8_000_000,game_fns=envs_fns,gamma=0.99,normalize_adv=False,decay_lr=False)
    ppo.run()

def train_dddq():
    def env_fn():
        return apply_wrappers_2(gym.make(id="ALE/Riverraid-v5"),framestack=4)

    dddq = DDDQ(env_fns= [env_fn for _ in range(8)],name="RiverFrameStack4")
    dddq.run()
def main():
    # train_pop3d()
    # train_recurrent_ppo()
    # train_impala()
    # train_a2c()
    # train_ppg()
    # train_ppo()
    # train_ppg_cartpole()
    # train_ppg_acrobat()
    # train_ppo_catrpole()
    # train_pop3d_cartpole()
    train_dddq()

if __name__ == "__main__":
    main()


import torch
import sys
import argparse

if __name__ != "Config":
    # if "Config" in sys.modules:
    #     class doubleImport(ImportError):
    #         """
    #         Config was imported using a different path after it has already been imported.
    #         This causes problems when Config is modified.
    #         """
    #         pass
    #     raise doubleImport
    print(f"A POSSIBLE PROBLEM HAS OCCURED, Config was loaded improperly, from {__name__} instead of directly\
    this might break some global variables by having two copies",file=sys.stderr)

#TODO: Rework config so that it is less janky and uses less bad practices of global variables.
# Possibly by moving HelperFunctions Loop functions to outside of the program
# and just using the command line parser for the individual sections.

#This config file is mainly used as global variables for the rest of the program.
#It should only be modified by the loop commands in helperfunctions

def loopOverUnknowns(unknownlist=False):
    """
    Given a list of unknowns (integers 0-14) this will create a list of knowns (the inverted list).
    """
    if unknownlist == False:
        unknownlist = parameters["Unknowns_clss"][0]
    knownVals = list(range(parameters["CLASSES"][0]))
    notused = unknownlist + UnusedClasses
    notused.sort()
    for un in notused:
        if un in knownVals:
            knownVals.remove(un)

    if len(unknownlist) > parameters["CLASSES"][0] -3:
        print("Too many unknowns, some algorithms might not work")
    if len(knownVals)<2:
        print("Too few knowns, things might break")
    parameters["Unknowns"] = f"{len(unknownlist)} Unknowns"
    parameters["Unknowns_clss"] = [unknownlist,"Values used for testing"]
    parameters["Knowns_clss"] = [knownVals,"Values used for training"]
    return knownVals

#This is the diffrent optimization functions
opt_func = {"Adam":torch.optim.Adam,"SGD":torch.optim.SGD, "RMSprop":torch.optim.RMSprop}

#Here are all of the paremeters for the model.
parameters = {
    #These parameters are orginized like this:
    #"ParamName":[Value,"Description",[possible values]]
    #for a parameter called "ParamName" with a value of Value
    "batch_size":[100000, "Number of items per batch"],
    "num_workers":[14, "Number of threads working on building batches"],
    "attemptLoad":[0, "0: do not use saves\n1:use saves"],
    "testlength":[1/4, "[0,1) percentage of training to test with"],
    "Mix unknowns and validation": [1,"0 or 1, 0 means that the test set is purely unknowns and 1 means that the testset is the validation set plus unknowns (for testing)"],
    "MaxPerClass": [1000, "Maximum number of samples per class\n if Dataloader_Variation is Cluster and this value is a float it interprets it as the maximum percentage of the class instead."],
    "num_epochs":[150,"Number of times it trains on the whole trainset"],
    "learningRate":[0.001, "a modifier for training"],
    "threshold":[0.5,"When to declare something to be unknown"],
    "model":["Convolutional","Model type",["Fully_Connected","Convolutional"]],
    "OOD Type":["Soft","type of out of distribution detection", ["Soft","Open","Energy","COOL","DOC","iiMod"]],
    "Dropout":[0.1,"percent of nodes that are skipped per run, larger numbers for more complex models [0,1)"],
    "Dataloader_Variation":["Standard","Defines the style of Dataloader used. This affects sampling from the dataset", ["Standard","Cluster","Flows"]],
    "optimizer":opt_func["Adam"],
    "Unknowns":["UNUSED"],
    "Unknowns_clss": [[7,8,9],"Class indexes used as unknowns."],
    "CLASSES":[15,"Number of classes, do not change"],
    "Temperature":[1,"Energy OOD scaling parameter"],
    "Degree of Overcompleteness": [3,"Parameter for Fitted Learning"],
    "Number of Layers": [3,"Number of layers to add to the base model"],
    "Nodes": [512,"The number of nodes per added layer"],
    "Activation": ["Leaky","The type of activation function to use",["ReLU", "Tanh", "Sigmoid","Leaky"]],
    "LOOP": [0,"This is a parameter that determines if we want to loop over the algorithms.\n "\
    "0: no loop, 1:loop through variations of algorithms,thresholds,learning rates, groups and numbers of epochs, \n"\
    "2: Loop while adding more unknowns into the training data (making them knowns) without resetting the model, \n"\
    "3: Loop through different data distributions without training the model.\n"\
    "4: Loop through predefined hyperparameters found in datasets/hyperparamList.csv"],
    "Dataset": ["Payload_data_CICIDS2017", "This is what dataset we are using,", ["Payload_data_CICIDS2017","Payload_data_UNSW"]],
    "SchedulerStepSize": [10, "This is how often the scheduler takes a step, 3 means every third epoch"],
    "SchedulerStep": [0.9,"This is how big a step the scheduler takes, leave 0 for no step"]
}

#Argparse tutorial: https://docs.python.org/3/howto/argparse.html
parser = argparse.ArgumentParser()
for x in parameters.keys():
    if x in ["batch_size","num_workers","MaxPerClass","num_epochs","Degree of Overcompleteness","Number of Layers","Nodes","SchedulerStepSize"]:
        parser.add_argument(f"--{x}",type=int,default=parameters[x][0],help=parameters[x][1],required=False)
    if x in ["testlength","learningRate","threshold","Dropout","Temperature","SchedulerStep"]:
        parser.add_argument(f"--{x}",type=float,default=parameters[x][0],help=parameters[x][1],required=False)
    if x in ["attemptLoad","Mix unknowns and validation"]:
        parser.add_argument(f"--{x}",type=int,choices=[1,0],default=parameters[x][0],help=parameters[x][1],required=False)
    if x in ["LOOP"]:
        parser.add_argument(f"--{x}",type=int,choices=[0,1,2,3,4],default=parameters[x][0],help=parameters[x][1],required=False)
    if x in ["model","OOD Type","Dataloader_Variation","Activation","Dataset"]:
        parser.add_argument(f"--{x}",choices=parameters[x].pop(),default=parameters[x][0],help=parameters[x][1],required=False)
    if x in ["Unknowns_clss"]:
        parser.add_argument(f"--{x}",default=f"{parameters[x][0]}",help=parameters[x][1],required=False)
if "pytest" not in sys.modules: #The argument parser appears to have issues with the pytest tests. I have no idea why.
    args = parser.parse_args()
    for x in args._get_kwargs():
        parameters[x[0]][0] = x[1]

if isinstance(parameters["Unknowns_clss"][0],str):
    if len(parameters["Unknowns_clss"][0])>0 and len(parameters["Unknowns_clss"][0])!=2: #Not sure why I need this specifier but it breaks if the default is []
        # print(len(parameters["Unknowns_clss"][0]))
        parameters["Unknowns_clss"][0] = [int(y) for y in parameters["Unknowns_clss"][0].removesuffix("]").removeprefix("[").split(sep=",")]
    else:
        parameters["Unknowns_clss"][0] = []

DOC_kernels = [3,4,5]

#Set Number of classes:
if parameters["Dataset"][0] == "Payload_data_UNSW":
    parameters["CLASSES"][0] = 10
    UnusedClasses = []
else:
    UnusedClasses = [8,9,10]
UnusedClasses = []

#Dendrogram chunk uses a slightly diffrent output on the model structure.
# (Also, dendrogram chunk is not working, so don't use it. Possibly related.)
if parameters["Dataloader_Variation"][0] == "Old_Cluster":
    parameters["CLASSES"][0] = parameters["CLASSES"][0] *32

#Add a value to the dictionary that is the inverse of the unknowns
loopOverUnknowns()

#This is for saving the original number of epochs
num_epochs = parameters["num_epochs"][0]

#This is to test all of the algorithms one after the other. (Loop 1 values)
alg = ["Soft","Open","Energy","COOL","DOC","iiMod"]
batch = [100,1000,10000,100000]
datapoints_per_class = [10,100,1000]
thresholds = [0.1,1,10]
thresholds = [30,20,15,5]
thresholds = [parameters["threshold"][0]]
learning_rates = [0.1,0.01,0.001,0.0001]
activation = ["ReLU", "Tanh", "Sigmoid","Leaky"]
groups = [[],[2],[2,3],[2,3,4],[2,3,4,5],[2,3,4,5,6],[2,3,4,5,6,7],[1,2,3,4,5,6,7],[1,2,3,4,5,6,7,8]]
#groups = [[7,8,9]]
if parameters["Dataset"][0] == "Payload_data_CICIDS2017":
    incGroups = [[2,3,4,5,6,7,8,9,10,11,12,13,14],[3,4,5,6,7,8,9,10,11,12,13,14],[4,5,6,7,8,9,10,11,12,13,14],[5,6,7,8,9,10,11,12,13,14],[6,7,8,9,10,11,12,13,14],[7,8,9,10,11,12,13,14],[8,9,10,11,12,13,14],[9,10,11,12,13,14],[10,11,12,13,14],[11,12,13,14],[12,13,14],[13,14],[14]]
#This one list is for loop 2. Note: array size should be decreasing.
else:
    incGroups = [[2,3,4,5,6,7,8,9],[3,4,5,6,7,8,9],[4,5,6,7,8,9],[5,6,7,8,9],[6,7,8,9],[7,8,9],[8,9],[9]]
epochs= []
epochs = [1,10,100,150]

# groups = [list(range(2,parameters["CLASSES"][0]))]
# #Little bit of code that generates incremental numbers of unknowns.
# while len(groups[0])>2:
#     new = groups[0].copy()
#     new.pop(0)
#     new.pop(0)
#     groups.insert(0,new)
# #Little bit of code that generates decrementing numbers of unknowns.
# incGroups = [list(range(2,parameters["CLASSES"][0]))]
# while len(incGroups[-1])>1:
#     new = incGroups[-1].copy()
#     new.pop(0)
#     incGroups.append(new)

#Here is where we remove some of the algorithms if we want to skip them. We could also just remove them from the list above.
# alg.remove("Soft")
# alg.remove("Open")
# alg.remove("Energy")
# alg.remove("COOL")
# alg.remove("DOC")
# alg.remove("iiMod")

#it causes problems if you dont start at the start of the loop.
if parameters["LOOP"][0] == 1:
    parameters["OOD Type"][0] = alg[0]

#Optimizer has been removed from the list of things we are changing
optim = [opt_func["Adam"], opt_func["SGD"], opt_func["RMSprop"]]
optim = [opt_func["Adam"]]

#Adds in everything in config:

# #learning_rates.remove(Config.parameters["learningRate"][0])
# learning_rates.insert(0,parameters["learningRate"][0])
# #epochs.remove(Config.parameters["num_epochs"][0])
# epochs.insert(0,parameters["num_epochs"][0])
# groups.insert(0,helper_variables["unknowns_clss"])

# #Always starts with the configured activation type
# alg.remove(parameters["OOD Type"][0])
# alg.insert(0,parameters["OOD Type"][0])

#This is an array to eaiser loop through everything.
loops = [batch,learning_rates,activation,["Standard","Cluster"],groups]
# loops = [groups]
loops2 = ["batch_size","learningRate","Activation","Dataloader_Variation","Unknowns"]
# loops2 = ["Unknowns"]
for i in range(len(loops)):
    if loops2[i] == "Unknowns":
        loops[i].insert(0,parameters["Unknowns_clss"][0])
    elif loops2[i] == "optimizer":
        loops[i].insert(0,parameters[loops2[i]])
    elif loops2[i] == "None":
        pass
    else:
        loops[i].insert(0,parameters[loops2[i]][0])

#Override the unknowns because model is kept
if parameters["LOOP"][0] == 2:
    parameters["Unknowns_clss"][0] = incGroups[0]
    parameters["Unknowns"] = f"{incGroups[0]} Unknowns"
    parameters["Knowns_clss"][0] = loopOverUnknowns()

#This controls all of the save data (so that we can run small tests without changing the nice files.)
unit_test_mode = False

use_alg_thesholds =False
def algorithmSpecificSettings(alg="None"):
    if use_alg_thesholds == False:
        return
    if alg == "None":
        alg = parameters["OOD Type"][0]

    # match alg:
    if alg == "Soft":
        pass
    if alg == "Open":
        parameters["threshold"][0] = 0.8
    if alg == "Energy":
        parameters["threshold"][0] = 0.474
    if alg == "COOL":
        parameters["threshold"][0] = 0.516034961
    if alg == "DOC":
        parameters["threshold"][0] = 0.06449493
    if alg == "iiMod":
        parameters["threshold"][0] = 102064.4453

if parameters["LOOP"][0] == 3:
    # parameters["num_epochs"][0] = 0
    parameters["loopLevel"] = [0,"What percentages the model is on"]
    parameters["MaxSamples"] = [parameters["MaxPerClass"][0], "Max number of samples total"]

#Getting version number
#https://gist.github.com/sg-s/2ddd0fe91f6037ffb1bce28be0e74d4e
f = open("build_number.txt","r")
parameters["Version"] = [f.read(),"The version number"]

save_as_tensorboard = True
datasetRandomOffset =True
dataparallel = True


import sys

import bottle
from bottlereact import BottleReact
from livereload import Server

PROD = False
DEBUG = True
RELOAD = True
LIVE_RELOAD = RELOAD
app = bottle.Bottle()
br = BottleReact(app, prod=PROD, verbose=True)

@app.get('/')
def root():
    return br.render_html(
        br.HelloWorld({'name': 'World'})
    )

@app.post('/login')
def login():
    return

def run():
    bottle.debug(DEBUG)
    if LIVE_RELOAD:
        server = Server(app)
        # server.watch
        server.serve(port=8081,
                     host='0.0.0.0')
    else:
        bottle.run(
            app=app,
            host='0.0.0.0',
            port='8081',
            reloader=RELOAD,
            server='auto'
        )

if __name__ == '__main__':
    run()


#######################################################################################################################
# Advent of Code 2022 - Day 16
#######################################################################################################################

import sys
import functools

sys.setrecursionlimit(10000)
STATE = {}

def read_input(file_str: str) -> None:
    """
    Read input about the valve config into state

    :param file_str: Location to input file as string
    """
    with open(file_str, "r") as input_file:
        for line in input_file:
            line = line.strip().split(" ")
            name = line[1]
            rate = int(line[4].split("=")[-1].split(";")[0])
            valves = [word.split(",")[0] for word in line[9:]]

            STATE[name] = {"rate": rate, "valves": valves}

@functools.cache
def part_one(open_valves: frozenset, mins_remaining: int, curr_valve: str) -> int:
    """
    Part 1 Solution:

    :param open_valves: Set of valves that have been opened and are releasing pressure each minute
    :param mins_remaining: Number of minutes remaining in the current function call (used recursively)
    :param curr_valve: Curr valve moved to in this function call (used recursively)
    :return: int - Most pressure that can be released
    """
    if mins_remaining <= 0:  # Base Case: No time left, so cannot open more valves
        return 0

    best = 0
    curr_valve_state = STATE[curr_valve]

    for valve in curr_valve_state["valves"]:
        best = max(best, part_one(open_valves, mins_remaining - 1, valve))

    if curr_valve not in open_valves and curr_valve_state["rate"] > 0 and mins_remaining > 0:
        open_valves = set(open_valves)
        open_valves.add(curr_valve)
        mins_remaining -= 1
        new_sum = mins_remaining * curr_valve_state["rate"]

        for valve in curr_valve_state["valves"]:
            best = max(best, new_sum + part_one(frozenset(open_valves), mins_remaining - 1, valve))

    return best

@functools.cache
def part_two(open_valves: frozenset, mins_remaining: int, curr_valve: str) -> int:
    """
    Part 2 Solution:
    Just do part one, then do it again once done to simulate the elephant (this is a naive solution, but it works)

    :param open_valves: Set of valves that have been opened and are releasing pressure each minute
    :param mins_remaining: Number of minutes remaining in the current function call (used recursively)
    :param curr_valve: Curr valve moved to in this function call (used recursively)
    :return: int - Most pressure that can be released
    """
    # Base Case: No time left, so cannot open more valves - instead, simulate elephant with the same 26 mins as well
    if mins_remaining <= 0:
        return part_one(open_valves, 26, "AA")

    best = 0
    curr_valve_state = STATE[curr_valve]

    for valve in curr_valve_state["valves"]:
        best = max(best, part_two(open_valves, mins_remaining - 1, valve))

    if curr_valve not in open_valves and curr_valve_state["rate"] > 0 and mins_remaining > 0:
        open_valves = set(open_valves)
        open_valves.add(curr_valve)
        mins_remaining -= 1
        new_sum = mins_remaining * curr_valve_state["rate"]

        for valve in curr_valve_state["valves"]:
            best = max(best, new_sum + part_two(frozenset(open_valves), mins_remaining - 1, valve))

    return best

def main():
    read_input("day16-input.txt")

    part1_sol = part_one(frozenset(), 30, 'AA')  # At start, we are in the room with 'AA' valve with 30 minutes to go
    print(f"Work out the steps to release the most pressure in 30 minutes."
          f"\nWhat is the most pressure you can release?"
          f"\nAnswer: {part1_sol}")

    part2_sol = part_two(frozenset(), 26, 'AA')  # At elephant training, in the room with 'AA' valve with 26 mins left
    print(f"With you and an elephant working together for 26 minutes, what is the most pressure you could release?"
          f"\nAnswer: {part2_sol}")

if __name__ == "__main__":
    main()


from django.shortcuts import render
from blog import models
from django.core.paginator import Paginator, EmptyPage,PageNotAnInteger #自动分页包

# Create your views here.

def index(request):
    fenlei = models.Category.objects.all()
    img = models.Banner.objects.filter(is_active=True)
    tags = models.Tag.objects.all()
    tuijian = models.Article.objects.filter(tui__id=1)[:3]
    tuijian2 = models.Article.objects.filter(tui__id=2)[:3]
    allarticle = models.Article.objects.all().order_by('-id')[0:4]
    Url = models.Link.objects.all()
    hot = models.Article.objects.all().order_by('-views')[:5]
                            # 通过浏览数进行排序,加个 - 由高到低排序
    content = {
        'A': fenlei,
        'B': img,
        'D': tags,
        'Url': Url,
        'E': tuijian,
        'F': tuijian2,
        'new': allarticle,
        'hot': hot
    }
    return render(request, 'index.html', content, )

#列表页
def list(request,lid):
    fenlei = models.Category.objects.all()
    weizhi1 = models.Category.objects.get(pk=lid)
    wenzhang = models.Article.objects.filter(category__id=lid).order_by('-id')
    tags = models.Tag.objects.all()
    tuijian = models.Article.objects.filter(tui__id=2)[:3]
    Tag = models.Tag.objects.filter(article__id=lid)
    hot = models.Article.objects.all().order_by('-views')[:5]
                            # 通过浏览数进行排序,加个 - 由高到低排序

    page = request.GET.get('page')  # 在URL中获取当前页面数
    paginator = Paginator(wenzhang, 2)  # 对查询到的数据对象list进行分页，设置超过5条数据就分页
    try:
        wenzhang = paginator.page(page)  # 获取当前页码的记录
    except PageNotAnInteger:
        wenzhang = paginator.page(1)  # 如果用户输入的页码不是整数时,显示第1页的内容
    except EmptyPage:
        wenzhang = paginator.page(paginator.num_pages)  # 如果用户输入的页数不在系统的页码列表中时,显示最后一页的内容

    content = {
        'A': fenlei,
        'weizhi': weizhi1,
        'C': wenzhang,

        'D': tags,
        'F': tuijian,
        'Tag': Tag,
        'hot': hot,
    }
    return render(request,'list.html',content,)

#内容页
def show(request,sid):
    fenlei = models.Category.objects.all() # 键A,base页面的，所有分类
    category_id = models.Article.objects.get(pk=sid).category_id  #取出文章的id所对应的的分类表id
    weizhi1 = models.Category.objects.get(pk=category_id) #利用文章id-分列表id取出文章所属的分类，键weizhi1 (您的位置： 首页 » weizhi1 » 正文)
    body = models.Article.objects.get(pk=sid) #show页面取出文章的具体内容，使用 {{ body.body|safe }} 取出富文本编辑器的内容
    tags = models.Tag.objects.all() #键D，侧边栏的所有标签
    tuijian = models.Article.objects.filter(tui__id=2)[:3]  #键F，侧边栏的热门推荐
    wenzhang = models.Article.objects.get(pk=sid)  #取出sid对应的文章所有信息
    fenlei2 = models.Article.objects.get(pk=sid).category #{{ weizhi1 }} 跳转链接用
    category_id2 = models.Article.objects.get(pk=sid).category_id   #根据文章id取出文章对应的分类ID
    tonglei = models.Article.objects.filter(category_id=category_id2)  #根据文章的分类ID，过滤和他同类的文章信息
    user_id = models.Article.objects.get(pk=sid).user_id  #取出文章的User_id
    user = models.User.objects.get(pk=user_id).username  #取出文章的具体作者
    Tag = models.Tag.objects.filter(article__id=sid)  #键Tag，取出文章所属于的所有标签
    body.views = body.views + 1 #对访问量 + 1
    body.save() #保存
    hot = models.Article.objects.all().order_by('-views')[:5]
                        #通过浏览数进行排序,加个 - 由高到低排序
    previous_blog = models.Article.objects.filter(created_time__gt=body.created_time, category=body.category.id).first()
    #前一偏
    next_blog = models.Article.objects.filter(created_time__lt=body.created_time, category=body.category.id).last()
    #后一偏

    content = {
        'A': fenlei,
        'weizhi1': weizhi1,
        'wenzhang': wenzhang,
        'body': body,
        'tonglei': tonglei,
        'D': tags,
        'F': tuijian,
        'fenlen2': fenlei2,
        'username': user,
        'Tag': Tag,
        'hot': hot,
        'syp': previous_blog,
        'next': next_blog,
    }
    return render(request,'show.html',content)

#标签页
def tag(request, tag):
    tag_id = models.Tag.objects.get(name=tag)
    fenlei = models.Category.objects.all()
    wenzhang = models.Article.objects.filter(tags=tag_id).order_by('-id')
    tags = models.Tag.objects.all()
    tuijian = models.Article.objects.filter(tui__id=2)[:3]
    hot = models.Article.objects.all().order_by('-views')[:5]

    page = request.GET.get('page')  # 在URL中获取当前页面数
    paginator = Paginator(wenzhang, 2)  # 对查询到的数据对象list进行分页，设置超过5条数据就分页
    try:
        wenzhang = paginator.page(page)  # 获取当前页码的记录
    except PageNotAnInteger:
        wenzhang = paginator.page(1)  # 如果用户输入的页码不是整数时,显示第1页的内容
    except EmptyPage:
        wenzhang = paginator.page(paginator.num_pages)  # 如果用户输入的页数不在系统的页码列表中时,显示最后一页的内容
    content = {
        'A': fenlei,
        'wenzhang': wenzhang,
        'D': tags,
        'F': tuijian,
        'tag_id': tag_id,
        'Tag': tag,
        'hot': hot,
    }
    return render(request,'tag.html',content)

# 搜索页
def search(request):
    fenlei = models.Category.objects.all()  # 键A,base页面的，所有分类
    guanjianci = request.GET.get('search') #获取搜索的关键词
    list = models.Article.objects.filter(title__icontains=guanjianci) #获取到搜索关键词公国标题比配
    remen = models.Article.objects.filter(tui__id=2)[:6]
    allcategory = models.Category.objects.all()
    page = request.GET.get('page')
    tags = models.Tag.objects.all()
    tuijian = models.Article.objects.filter(tui__id=2)[:3]
    hot = models.Article.objects.all().order_by('-views')[:5]
    paginator = Paginator(list, 2) #每一页最多显示的文章数量
    try:
        list = paginator.page(page) # 获取当前页码的记录
    except PageNotAnInteger:
        list = paginator.page(1)  # 如果用户输入的页码不是整数时,显示第1页的内容
    except EmptyPage:
        list = paginator.page(paginator.num_pages)   # 如果用户输入的页数不在系统的页码列表中时,显示最后一页的内容

    content = {
        'A': fenlei,
        'guanjianci': guanjianci,
        'list': list, #分页用
        'remen': remen,
        'allcategory': allcategory,
        'page': page,
        'D': tags,
        'F': tuijian,
        'hot': hot,
    }
    return render(request, 'search.html', content)

# 关于我们
def about(request):
    fenlei = models.Category.objects.all()
    content = {
        'A': fenlei,
    }
    return render(request,'page.html',content)


"""
Module contains variable definitions and misc utility functions.

Functions and logic inspired/imported from https://github.com/THGLab/X-EISD/
"""
import os
import numpy as np

exp_idx = 'index'
exp_resnum = 'resnum'
exp_val = 'value'
exp_max = 'upper'
exp_atmID = 'atomname'
exp_min = 'lower'
exp_err = 'error'

eisd_run_all = 'all'
eisd_run_single = 'single'
eisd_run_pairs = 'pairs'
eisd_modes = (
    eisd_run_all,
    eisd_run_single,
    eisd_run_pairs
    )
default_mode = eisd_run_all

opt_max = 'max'
opt_mc = 'mc'
eisd_optimization_types = (
    opt_max,
    opt_mc,
    None,
    )
default_type = opt_max

parse_mode_exp = 'exp'
parse_mode_back = 'bc'
parse_modes = (
    parse_mode_exp,
    parse_mode_back,
    )

saxs_name = 'saxs'
cs_name = 'cs'
fret_name = 'fret'
jc_name = 'jc'
noe_name = 'noe'
pre_name = 'pre'
rdc_name = 'rdc'
rh_name = 'rh'
eisd_modules = (
    saxs_name,
    cs_name,
    fret_name,
    jc_name,
    noe_name,
    pre_name,
    rdc_name,
    rh_name,
    )

# define back calculation uncertainties
# refer to Lincoff et al. 2020 for details
default_bc_errors = {
    pre_name: 0.0001,
    noe_name: 0.0001,
    saxs_name: 0.006,
    fret_name: 0.0074,
    rh_name: 0.812,
    rdc_name: 0.88,
    # cs error reported from UCBShift
    cs_name: {'C': 1.31, 'CA': 0.97, 'CB': 1.29, 'H': 0.38, 'HA': 0.29},
    jc_name: {'A': np.sqrt(0.14), 'B': np.sqrt(0.03), 'C': np.sqrt(0.08)},
    }
jc_bc_mu = {'A': 6.51, 'B': -1.76, 'C': 1.6}

rmse_idx = 0
score_idx = 1
avg_bc_idx = 2

XEISD_TITLE = (
    "\n _  _     ____  ____  ___  ____  \n"  # noqa: W605
    "( \/ )___( ___)(_  _)/ __)(  _ \ \n"  # noqa: W605
    " )  ((___))__)  _)(_ \__ \ )(_) )\n"  # noqa: W605
    "(_/\_)   (____)(____)(___/(____/ \n"  # noqa: W605
    "================================="
    )

def add_optimization_mode(parser):
    """Add option to choose from optimization methods."""
    parser.add_argument(
        '-m',
        '--mode',
        help=(
            'Optimization mode. Could be Metropolis Monte Carlo (mc) '
            'or score maximization method (max).'
            'Defaults to max.'
            ),
        default=opt_max,
        choices=tuple((opt_max, opt_mc)),
        )

# The following two functions have been imported from:
# https://github.com/THGLab/X-EISD/blob/master/eisd/utils/miscell.py
# https://github.com/Oufan75/X-EISD/blob/master/eisd/utils.py
def make_pairs(all):
    """Make pairs of items in a list."""
    pairs = []
    for i in range(len(all)):
        for j in range(i + 1, len(all)):
            pairs.append([all[i], all[j]])
    return pairs

def modes(mode, all):
    """Initialize a mode from a list."""
    flags = {}
    for prop in all:
        flags[prop] = False

    if mode is eisd_run_all:
        return {flag: True for flag in flags}
    elif type(mode) is list:
        for flag in mode:
            flags[flag] = True
        return flags
    elif type(mode) is str:
        flags[mode] = True
        return flags

    else:
        raise ValueError("The mode in the main function is not recognized.")

def meta_data(fpath):
    """
    Filter through exp and bc ata files.

    Return only the paths that exist in both cases.

    Automatically removes paths to datafiles if e.g. there are more
    back-calculated data than experimental.

    Parameters
    ----------
    fpath : str or Path
        Path to the directory containing both experimental
        and back-calculated data files.

    Returns
    -------
    meta : dict
        First layer keys of `exp` and `bc` having values of dictionaries
        with keys being the module and the value being the path to the data.

    errlog : list
        List of errors to relay to the user if there are any.
    """
    exp_paths = []
    back_paths = []
    valid_exp_modules = []
    valid_back_modules = []

    meta = {}
    tobc = None
    errlog = []

    all_files = [f for f in os.listdir(fpath) if os.path.isfile(os.path.join(fpath, f))]  # noqa: E501
    for f in all_files:
        if f.startswith(parse_mode_exp):
            if f.endswith(eisd_modules):
                exp_paths.append(os.path.join(fpath, f))
                _ext = f[f.rindex('.') + 1:]
                valid_exp_modules.append(f'.{_ext}')
        elif f.startswith(parse_mode_back):
            if f.endswith(eisd_modules):
                back_paths.append(os.path.join(fpath, f))
                _ext = f[f.rindex('.') + 1:]
                valid_back_modules.append(f'.{_ext}')

    valid_exp_modules.sort()
    valid_back_modules.sort()

    if valid_exp_modules == []:
        errlog.append(
            'WARNING: no valid experimental files found.'
            ' Please refer to the help documentation for'
            ' this module.'
            )
        return meta, errlog
    else:
        differences = tuple(set(valid_exp_modules) ^ (set(valid_back_modules)))

        # Do not clear `exp_paths` because we do back-calculations internally
        if differences:
            #exp_paths = [exp for exp in exp_paths if not exp.endswith(differences)]  # noqa: E265, E501
            back_paths = [bck for bck in back_paths if not bck.endswith(differences)]  # noqa: E501
            errlog.append(
                'Note: found inconsistent experimental and back-calculation'
                ' data pairs.'
                )
            errlog.append(f'Back-calculating for: {differences}...')
            tobc = [module[1:] for module in differences]

    EXP_DATA_FILENAMES = {}
    BACK_DATA_FILENAMES = {}

    for module in eisd_modules:
        for exp in exp_paths:
            if module in exp:
                EXP_DATA_FILENAMES[module] = exp
        for bc in back_paths:
            if module in bc:
                BACK_DATA_FILENAMES[module] = bc

    meta = {
        parse_mode_exp: EXP_DATA_FILENAMES,
        parse_mode_back: BACK_DATA_FILENAMES,
        }

    return meta, tobc, errlog


"""Platform for sensor integration."""
from __future__ import annotations

import logging
from datetime import timedelta
from typing import Any

from homeassistant.components.sensor import SensorDeviceClass
from homeassistant.components.sensor import SensorEntity
from homeassistant.config_entries import ConfigEntry
from homeassistant.core import callback
from homeassistant.core import HomeAssistant
from homeassistant.helpers.entity import DeviceInfo
from homeassistant.helpers.entity_platform import AddEntitiesCallback
from homeassistant.helpers.update_coordinator import CoordinatorEntity
from homeassistant.helpers.update_coordinator import DataUpdateCoordinator

from .const import ALARM_ID
from .const import DOMAIN
from .const import HINT
from .const import PRIORITY
from .const import REGION
from .const import SIGNATURE
from .const import TITLE
from .const import VALID_FROM
from .const import VALID_TO
from .wetter_alarm_client import WetterAlarmApiClient

_LOGGER = logging.getLogger(__name__)
SCAN_INTERVAL = timedelta(seconds=60)

async def async_setup_entry(
    hass: HomeAssistant,
    config_entry: ConfigEntry,
    async_add_entities: AddEntitiesCallback,
):
    pois_from_config = config_entry.data["pois"]
    all_sensors = []
    for poi_name, poi_id in pois_from_config:
        coordinator = WetterAlarmCoordinator(
            hass=hass, logger=_LOGGER, poi_name=poi_name, poi_id=poi_id
        )
        sensors = [
            WetterAlarmIdSensor(coordinator, ALARM_ID),
            WetterAlarmValidFromSensor(coordinator, VALID_FROM),
            WetterAlarmValidToSensor(coordinator, VALID_TO),
            WetterAlarmPrioritySensor(coordinator, PRIORITY),
            WetterAlarmRegionSensor(coordinator, REGION),
            WetterAlarmTitleSensor(coordinator, TITLE),
            WetterAlarmHintSensor(coordinator, HINT),
            WetterAlarmSignatureSensor(coordinator, SIGNATURE),
        ]
        all_sensors.extend(sensors)
        await coordinator.async_config_entry_first_refresh()
    async_add_entities(all_sensors)

class WetterAlarmBaseSensor(CoordinatorEntity, SensorEntity):
    """base Wetter-Alarm Sensor, all sensors inherit from it"""

    def __init__(self, coordinator: WetterAlarmCoordinator, suffix: str):
        SensorEntity.__init__(self)
        CoordinatorEntity.__init__(self, coordinator)
        self._name = coordinator.name
        self._suffix = suffix
        self._poi_id = coordinator.get_poi_id
        _LOGGER.debug(f"created Sensor of class {__class__} with uid {self.unique_id}")

    _attr_has_entity_name: True
    _attr_should_poll: True

    @property
    def name(self) -> str | None:
        return f"{self._name} {self._suffix}"

    @property
    def friendly_name(self) -> str | None:
        return f"{self._name} {self._suffix.replace('_', ' ').capitalize()}"

    @property
    def device_info(self) -> DeviceInfo:
        """Return the device info."""
        info = DeviceInfo(
            identifiers={(DOMAIN, str(self._poi_id))},
            name=f"Point of Interest-{self._name}/{self._poi_id}",
            default_manufacturer="Wetter-Alarm",
            default_model="API",
        )
        return info

    @property
    def unique_id(self) -> str | None:
        return f"Point of Interest - {self._poi_id} - {self._suffix}"

    @callback
    def _handle_coordinator_update(self) -> None:
        """Handle updated data from the coordinator."""
        value = self.coordinator.data[self._suffix]
        self._attr_native_value = value
        self.async_write_ha_state()

class WetterAlarmIdSensor(WetterAlarmBaseSensor):
    """Sensor for the Alarm ID"""

    _attr_icon = "mdi:identifier"

class WetterAlarmValidFromSensor(WetterAlarmBaseSensor):
    """Sensor for when the alarm starts"""

    _attr_icon = "mdi:calendar-arrow-left"
    _attr_device_class = SensorDeviceClass.DATE

class WetterAlarmValidToSensor(WetterAlarmBaseSensor):
    """Sensor for when the alarm ends"""

    _attr_icon = "mdi:calendar-arrow-right"
    _attr_device_class = SensorDeviceClass.DATE

class WetterAlarmPrioritySensor(WetterAlarmBaseSensor):
    """Sensor for the Alarm priority"""

    _attr_icon = "mdi:chevron-triple-up"

class WetterAlarmRegionSensor(WetterAlarmBaseSensor):
    """Sensor for the region the alarm occurs in"""

    _attr_icon = "mdi:map-marker-check-outline"

class WetterAlarmTitleSensor(WetterAlarmBaseSensor):
    """Sensor for the title of the Alarm"""

    _attr_icon = "mdi:format-title"

class WetterAlarmHintSensor(WetterAlarmBaseSensor):
    """Sensor for the Alarm hint"""

    _attr_icon = "mdi:account-alert"

class WetterAlarmSignatureSensor(WetterAlarmBaseSensor):
    """Sensor for who issued the alarm"""

    _attr_icon = "mdi:signature-freehand"

class WetterAlarmCoordinator(DataUpdateCoordinator):
    """Custom Wetter-Alarm Coordinator"""

    def __init__(
        self, hass: HomeAssistant, logger: logging.Logger, poi_id: int, poi_name: str
    ) -> None:
        self._hass = hass
        self._poi_id = poi_id
        self._name = f"{poi_name}"
        self._logger = logger

        super().__init__(
            hass=hass,
            logger=logger,
            name=self._name,
            update_interval=SCAN_INTERVAL,
        )

    @property
    def get_hass(self):
        return self._hass

    @property
    def get_poi_id(self):
        return self._poi_id

    async def _async_update_data(self) -> dict[str, Any]:
        """Fetch data from API endpoint.
        This is the place to pre-process the data to lookup tables
        so entities can quickly look up their data.
        """

        self.update_interval = SCAN_INTERVAL

        async def fetch_all_values() -> dict[str, float]:
            client = WetterAlarmApiClient(self._poi_id)
            data = await client.search_for_alerts_async(hass=self._hass)
            return data

        return await fetch_all_values()


# -*- coding: utf-8 -*-
# @Author: WuLC
# @Date:   2016-12-23 16:43:34
# @Last modified by:   WuLC
# @Last Modified time: 2016-12-23 17:03:35
# @Email: liangchaowu5@gmail.com

# sort the hashmap
class Solution(object):
    def frequencySort(self, s):
        """
        :type s: str
        :rtype: str
        """
        result = ''
        count = {}
        for char in s:
            count.setdefault(char, 0)
            count[char] += 1
        sorted_dict = sorted(count.items(), key = lambda x:x[1], reverse = True)
        for pair in sorted_dict:
            result += pair[0]*pair[1]
        return result

Algorithm/Python/451. Sort Characters By Frequency.py


"""Some common spatial plotting routines. Useful for contextualizing nanoARPES data."""
from typing import Any, Dict, List, Optional
import itertools

import numpy as np
import xarray as xr

import matplotlib.pyplot as plt
import matplotlib.patches as patches
from matplotlib import cm
import matplotlib.patheffects as path_effects
import matplotlib.gridspec as gridspec

from arpes.io import load_data
from arpes.plotting.annotations import annotate_point
from arpes.plotting.utils import (
    path_for_plot,
    frame_with,
    remove_colorbars,
    fancy_labels,
    ddata_daxis_units,
)
from arpes.provenance import save_plot_provenance
from arpes.typing import DataType
from arpes.utilities import normalize_to_spectrum
from arpes.utilities.xarray import unwrap_xarray_item

__all__ = ("reference_scan_spatial", "plot_spatial_reference")

@save_plot_provenance
def plot_spatial_reference(
    reference_map: DataType,
    data_list: List[DataType],
    offset_list: Optional[List[Dict[str, Any]]] = None,
    annotation_list: Optional[List[str]] = None,
    out: Optional[str] = None,
    plot_refs: bool = True,
):
    """Helpfully plots data against a reference scanning dataset.

    This is essential to understand
    where data was taken and can be used early in the analysis phase in order to highlight the
    location of your datasets against core levels, etc.

    Args:
        reference_map: A scanning photoemission like dataset
        data_list: A list of datasets you want to plot the relative locations of
        offset_list: Optionally, offsets given as coordinate dicts
        annotation_list: Optionally, text annotations for the data
        out: Where to save the figure if we are outputting to disk
        plot_refs: Whether to plot reference figures for each of the pieces of data in `data_list`
    """
    if offset_list is None:
        offset_list = [None] * len(data_list)

    if annotation_list is None:
        annotation_list = [str(i + 1) for i in range(len(data_list))]

    normalize_to_spectrum(reference_map)

    n_references = len(data_list)
    if n_references == 1 and plot_refs:
        fig, axes = plt.subplots(1, 2, figsize=(12, 5))
        ax = axes[0]
        ax_refs = [axes[1]]
    elif plot_refs:
        n_extra_axes = 1 + (n_references // 4)
        fig = plt.figure(figsize=(6 * (1 + n_extra_axes), 5))
        spec = gridspec.GridSpec(ncols=2 * (1 + n_extra_axes), nrows=2, figure=fig)
        ax = fig.add_subplot(spec[:2, :2])

        ax_refs = [
            fig.add_subplot(spec[i // (2 * n_extra_axes), 2 + i % (2 * n_extra_axes)])
            for i in range(n_references)
        ]
    else:
        ax_refs = []
        fig, ax = plt.subplots(1, 1, figsize=(6, 5))

    try:
        reference_map = reference_map.S.spectra[0]
    except Exception:
        pass

    reference_map = reference_map.S.mean_other(["x", "y", "z"])

    ref_dims = reference_map.dims[::-1]

    assert len(reference_map.dims) == 2
    reference_map.S.plot(ax=ax, cmap="Blues")

    cmap = cm.get_cmap("Reds")
    rendered_annotations = []
    for i, (data, offset, annotation) in enumerate(zip(data_list, offset_list, annotation_list)):
        if offset is None:
            try:
                offset = data.S.logical_offsets - reference_map.S.logical_offsets
            except ValueError:
                offset = {}

        coords = {c: unwrap_xarray_item(data.coords[c]) for c in ref_dims}
        n_array_coords = len(
            [cv for cv in coords.values() if isinstance(cv, (np.ndarray, xr.DataArray))]
        )
        color = cmap(0.4 + (0.5 * i / len(data_list)))
        x = coords[ref_dims[0]] + offset.get(ref_dims[0], 0)
        y = coords[ref_dims[1]] + offset.get(ref_dims[1], 0)
        ref_x, ref_y = x, y
        off_x, off_y = 0, 0
        scale = 0.03

        if n_array_coords == 0:
            off_y = 1
            ax.scatter([x], [y], s=60, color=color)
        if n_array_coords == 1:
            if isinstance(x, (np.ndarray, xr.DataArray)):
                y = [y] * len(x)
                ref_x = np.min(x)
                off_x = -1
            else:
                x = [x] * len(y)
                ref_y = np.max(y)
                off_y = 1

            ax.plot(x, y, color=color, linewidth=3)
        if n_array_coords == 2:
            off_y = 1
            min_x, max_x = np.min(x), np.max(x)
            min_y, max_y = np.min(y), np.max(y)
            ref_x, ref_y = min_x, max_y

            color = cmap(0.4 + (0.5 * i / len(data_list)), alpha=0.5)
            rect = patches.Rectangle((min_x, min_y), max_x - min_x, max_y - min_y, facecolor=color)
            color = cmap(0.4 + (0.5 * i / len(data_list)))

            ax.add_patch(rect)

        dp = ddata_daxis_units(ax)
        text_location = (
            np.asarray(
                [
                    ref_x,
                    ref_y,
                ]
            )
            + dp * scale * np.asarray([off_x, off_y])
        )
        text = ax.annotate(annotation, text_location, color="black", size=15)
        rendered_annotations.append(text)
        text.set_path_effects(
            [path_effects.Stroke(linewidth=2, foreground="white"), path_effects.Normal()]
        )
        if plot_refs:
            ax_ref = ax_refs[i]
            keep_preference = list(ref_dims) + [
                "eV",
                "temperature",
                "kz",
                "hv",
                "kp",
                "kx",
                "ky",
                "phi",
                "theta",
                "beta",
                "pixel",
            ]
            keep = [d for d in keep_preference if d in data.dims][:2]
            data.S.mean_other(keep).S.plot(ax=ax_ref)
            ax_ref.set_title(annotation)
            fancy_labels(ax_ref)
            frame_with(ax_ref, color=color, linewidth=3)

    ax.set_title("")
    remove_colorbars()
    fancy_labels(ax)
    plt.tight_layout()

    try:
        from adjustText import adjust_text

        adjust_text(
            rendered_annotations,
            ax=ax,
            avoid_points=False,
            avoid_objects=False,
            avoid_self=False,
            autoalign="xy",
        )
    except ImportError:
        pass

    if out is not None:
        plt.savefig(path_for_plot(out), dpi=400)
        return path_for_plot(out)

    return fig, [ax] + ax_refs

@save_plot_provenance
def reference_scan_spatial(data, out=None, **kwargs):
    """Plots the spatial content of a dataset, useful as a quick reference."""
    data = normalize_to_spectrum(data)

    dims = [d for d in data.dims if d in {"cycle", "phi", "eV"}]

    summed_data = data.sum(dims, keep_attrs=True)

    fig, ax = plt.subplots(3, 2, figsize=(15, 15))
    flat_axes = list(itertools.chain(*ax))

    summed_data.plot(ax=flat_axes[0])
    flat_axes[0].set_title(r"Full \textbf{eV} range")

    dims_except_eV = [d for d in dims if d != "eV"]
    summed_data = data.sum(dims_except_eV)

    mul = 0.2
    rng = data.coords["eV"].max().item() - data.coords["eV"].min().item()
    offset = data.coords["eV"].max().item()
    if offset > 0:
        offset = 0

    if rng > 3:
        mul = rng / 5.0

    for i in range(5):
        low_e, high_e = -mul * (i + 1) + offset, -mul * i + offset
        title = r"\textbf{eV}" + ": {:.2g} to {:.2g}".format(low_e, high_e)
        summed_data.sel(eV=slice(low_e, high_e)).sum("eV").plot(ax=flat_axes[i + 1])
        flat_axes[i + 1].set_title(title)

    y_range = flat_axes[0].get_ylim()
    x_range = flat_axes[0].get_xlim()
    delta_one_percent = ((x_range[1] - x_range[0]) / 100, (y_range[1] - y_range[0]) / 100)

    smart_delta = (2 * delta_one_percent[0], -1.5 * delta_one_percent[0])

    referenced = data.S.referenced_scans

    # idea here is to collect points by those that are close together, then
    # only plot one annotation
    condensed = []
    cutoff = 3  # 3 percent
    for index, _ in referenced.iterrows():
        ff = load_data(index)

        x, y, _ = ff.S.sample_pos
        found = False
        for cx, cy, cl in condensed:
            if abs(cx - x) < cutoff * abs(delta_one_percent[0]) and abs(cy - y) < cutoff * abs(
                delta_one_percent[1]
            ):
                cl.append(index)
                found = True
                break

        if not found:
            condensed.append((x, y, [index]))

    for fax in flat_axes:
        for cx, cy, cl in condensed:
            annotate_point(
                fax,
                (
                    cx,
                    cy,
                ),
                ",".join([str(l) for l in cl]),
                delta=smart_delta,
                fontsize="large",
            )

    plt.tight_layout()

    if out is not None:
        plt.savefig(path_for_plot(out), dpi=400)
        return path_for_plot(out)

    return fig, ax


import pygame
from .vector import *
from .bitmap_font import *

# Initially loads the sprites into memory
# Can be called multiple times to reload them
def reload_sprites( self ):

    self._Engine__sprites = {}

    spr_file = open( self.get_path( '/textures/list.txt' ) ).read().split( '\n' )

    # Iterate through every line in the sprite file
    # Sprites are listed as follows:
    # [name] = [relative filepath] [scale]:[# vertical subimages]:[# horizontal subimages]
    for line in spr_file:

        # Parse through the data within the line of text
        internal_name, line = line.split( ' = ' )
        filename, line = line.split( ' ' )
        dimensions = [ int( a ) for a in line.split( ':' ) ]

        # Load the image and transform it based on the data in the file line
        surface = pygame.image.load( self.get_path( '/textures/' + filename ) )
        surface = pygame.transform.scale( surface, V2( surface.get_size() ).l() )

        # Use the dimensions of the sprite divided by the # of subimages to get the size of a square
        dims = V2( surface.get_size() )
        square_count = V2( dimensions )
        square_size = dims.c().d( square_count )

        # Use the previous information to split the sprite up and append it to the sprite data
        self._Engine__sprites[ internal_name ] = [ [ surface.subsurface( ( xx * square_size.x, yy * square_size.y, *square_size.l() ) ) for xx in range( square_count.x ) ]
            for yy in range( square_count.y ) ]

def draw_line( self, pos_a, pos_b, is_ui, color, **kwargs ):

    # "is_ui" specifies whether to use game coordinates or UI coordinates
    # Game coords are measured in blocks and UI coords are measured in pixels
    if is_ui:
        pos_a = pos_a.c()
        pos_b = pos_b.c()
    else:
        pos_a = self.to_screen_coord( pos_a )
        pos_b = self.to_screen_coord( pos_b )

    # Draw antialiased line if told to
    if 'is_aa' in kwargs and kwargs[ 'is_aa' ]:
        pygame.draw.aaline( self._Engine__screen, color, pos_a.l(), pos_b.l() )
    # Otherwise, draw jagged line
    else:
        pygame.draw.line( self._Engine__screen, color, pos_a.l(), pos_b.l() )

# Shortcut for blitting transformed surface onto screen
# Every other draw function (except draw_line) leads to this one
def draw_surface( self, surf, pos, is_ui, **kwargs ):

    # "is_ui" specifies whether to use game coordinates or UI coordinates
    # Game coords are measured in blocks and UI coords are measured in pixels
    if is_ui:
        pos = pos.c()
    else:
        pos = self.to_screen_coord( pos )

    # Scale based on zoom level (doesn't affect UI)
    # Can also be disabled via 'fixed_size = True'
    if not ( is_ui or ( 'fixed_size' in kwargs and kwargs[ 'fixed_size' ] ) ):

        # Use an already-existing surface if possible
        if 'buffer_key' in kwargs and kwargs[ 'buffer_key' ] in self._Engine__zoom_buffer:
            surf = self._Engine__zoom_buffer[ kwargs[ 'buffer_key' ] ]
        else:
            surf = pygame.transform.scale( surf, V2( surf.get_size() ).m( self.view_zoom ).l() )

    # Store the scaled version of this surface in memory if told to
    # The caller of the function is in charge of providing a unique key
    if ( 'buffer_key' in kwargs and kwargs[ 'buffer_key' ] not in self._Engine__zoom_buffer ):
        self._Engine__zoom_buffer[ kwargs[ 'buffer_key' ] ] = surf

    # Check each transformation argument to see if sprite needs to be modified
    if ( 'scale' in kwargs ):
        surf = pygame.transform.scale( surf, V2( surf.get_size() ).m( kwargs[ 'scale' ] ).l() )
    if ( 'flip' in kwargs ):
        surf = pygame.transform.flip( surf, kwargs[ 'flip' ].x == -1, kwargs[ 'flip' ].y == -1 )
    if ( 'rotation' in kwargs ):
        surf = pygame.transform.rotate( surf, kwargs[ 'rotation' ] )

    # Shift the position based off of the anchor
    anchor = V2() if 'anchor' not in kwargs else kwargs[ 'anchor' ]
    pos.s( anchor.c().m( surf.get_size() ) )

    # Draw the modified surface
    self._Engine__screen.blit( surf, pos.l() )

# Uses pre-defined surface
def draw_sprite( self, sprite_id, frame, pos, is_ui, **kwargs ):

    sprite_surf = self._Engine__sprites[ sprite_id ][ frame.x ][ frame.y ]
    self.draw_surface( sprite_surf, pos, is_ui, **kwargs )

# Draws non-bitmap text onto the screen
def draw_text( self, text, font, pos, is_ui, color = ( 255, 255, 255 ), **kwargs ):

    text_surf = self._Engine__fonts[ font ].render( text, True, color )
    self.draw_surface( text_surf, pos, is_ui, **kwargs )

# Draws bitmap text (requires bitmap font)
def draw_text_bitmap( self, text, bitmap_font, scale, pos, is_ui, color = ( 255, 255, 255 ), sep = 1, **kwargs ):

    text_surf = self._Engine__bitmap_fonts[ bitmap_font ].render( text, scale, color, sep )

    # Change the color if necessary (from reddit.com/r/pygame/comments/hprkpr/how_to_change_the_color_of_an_image_in_pygame/)
    if ( color != ( 255, 255, 255 ) ):
        text_array = pygame.PixelArray( text_surf )
        text_array.replace( ( 255, 255, 255, 255 ), ( *color, 255 ) if len( color ) == 3 else color )
        del text_array

    self.draw_surface( text_surf, pos, is_ui, **kwargs )

# Converts game coordinates to screen coordinates
# Basically shifts it relative to view
def to_screen_coord( self, world_pos ):

    # zoomed_position = ( world_position - center_position ) * zoom_level
    zoomed_pos = world_pos.c().s( self.view_pos ).m( self.view_zoom ).i()

    # screen_position = zoomed_position + half_size
    screen_pos = zoomed_pos.a( self.screen_size.c().d( 2 ) )

    return screen_pos

# Undoes to_screen_coord
def to_world_coord( self, screen_pos ):

    # zoomed_position = screen_position - half_size
    zoomed_pos = screen_pos.c().s( self.screen_size.c().d( 2 ) )

    # world_position = ( zoomed_position / zoom_level ) + center_position
    world_pos = zoomed_pos.d( self.view_zoom ).a( self.view_pos )

    return world_pos

# Removes a sprite with a specific key from the zoom buffer
def zoom_buffer_remove( self, key ):

    if key in self._Engine__zoom_buffer:
        self._Engine__zoom_buffer.pop( key )

# Returns a sprite surface for other objects to use
def get_sprite( self, sprite_id, frame ):

    return self._Engine__sprites[ sprite_id ][ frame.x ][ frame.y ]

# Creates a font under the name 'name:size'
# Loads it from an external .ttf or .otf file
def create_font( self, filepath, name, size ):

    self._Engine__fonts[ f'{ name }:{ size }' ] = pygame.font.Font( self.get_path( filepath ), size )

# Creates a bitmap font from a .PNG
def create_bitmap_font( self, filepath, name, space_width = None ):

    self._Engine__bitmap_fonts[ name ] = Bitmap_Font( self.get_path( filepath ), space_width = space_width )


"""
Corpus loading worker
---------------------
"""
from __future__ import annotations

import os
import threading
import typing
from pathlib import Path
from queue import Empty, Queue

import sqlalchemy
from sqlalchemy.orm import joinedload, subqueryload

from montreal_forced_aligner.abc import KaldiFunction
from montreal_forced_aligner.corpus.classes import FileData
from montreal_forced_aligner.corpus.helper import find_exts
from montreal_forced_aligner.data import Language, MfaArguments
from montreal_forced_aligner.db import Dictionary, Job, Speaker, Utterance
from montreal_forced_aligner.exceptions import SoundFileError, TextGridParseError, TextParseError
from montreal_forced_aligner.helper import mfa_open
from montreal_forced_aligner.utils import Counter

if typing.TYPE_CHECKING:
    from dataclasses import dataclass

    from montreal_forced_aligner.models import G2PModel
    from montreal_forced_aligner.tokenization.simple import SimpleTokenizer

    try:
        from spacy.language import Language as SpacyLanguage
    except ImportError:
        SpacyLanguage = None
else:
    from dataclassy import dataclass

__all__ = [
    "AcousticDirectoryParser",
    "CorpusProcessWorker",
    "ExportKaldiFilesFunction",
    "ExportKaldiFilesArguments",
    "NormalizeTextFunction",
    "NormalizeTextArguments",
]

def dictionary_ids_for_job(session, job_id):
    dictionary_ids = [
        x[0]
        for x in session.query(Dictionary.id)
        .join(Utterance.speaker)
        .join(Speaker.dictionary)
        .filter(Utterance.in_subset == True)  # noqa
        .filter(Utterance.job_id == job_id)
        .distinct()
    ]
    return dictionary_ids

class AcousticDirectoryParser(threading.Thread):
    """
    Worker for processing directories for acoustic sound files

    Parameters
    ----------
    corpus_directory: str
        Directory to parse
    job_queue: Queue
        Queue to add file names to
    audio_directory: str
        Directory with additional audio files
    stopped: :class:`~threading.Event`
        Check for whether to exit early
    finished_adding: :class:`~threading.Event`
        Check to set when the parser is done adding files to the queue
    file_counts: :class:`~montreal_forced_aligner.utils.Counter`
        Counter for the number of total files that the parser has found
    """

    def __init__(
        self,
        corpus_directory: str,
        job_queue: Queue,
        audio_directory: str,
        stopped: threading.Event,
        finished_adding: threading.Event,
        file_counts: Counter,
    ):
        super().__init__()
        self.corpus_directory = corpus_directory
        self.job_queue = job_queue
        self.audio_directory = audio_directory
        self.stopped = stopped
        self.finished_adding = finished_adding
        self.file_counts = file_counts

    def run(self) -> None:
        """
        Run the corpus loading job
        """

        use_audio_directory = False
        all_sound_files = {}
        if self.audio_directory and os.path.exists(self.audio_directory):
            use_audio_directory = True
            for root, _, files in os.walk(self.audio_directory, followlinks=True):
                exts = find_exts(files)
                wav_files = {k: os.path.join(root, v) for k, v in exts.wav_files.items()}
                other_audio_files = {
                    k: os.path.join(root, v) for k, v in exts.other_audio_files.items()
                }
                all_sound_files.update(other_audio_files)
                all_sound_files.update(wav_files)
        for root, _, files in os.walk(self.corpus_directory, followlinks=True):
            exts = find_exts(files)
            relative_path = root.replace(str(self.corpus_directory), "").lstrip("/").lstrip("\\")

            if self.stopped.is_set():
                break
            if not use_audio_directory:
                all_sound_files = {}
                exts.wav_files = {k: os.path.join(root, v) for k, v in exts.wav_files.items()}
                exts.other_audio_files = {
                    k: os.path.join(root, v) for k, v in exts.other_audio_files.items()
                }
                all_sound_files.update(exts.other_audio_files)
                all_sound_files.update(exts.wav_files)
            for file_name in exts.identifiers:
                if self.stopped.is_set():
                    break
                wav_path = None
                transcription_path = None
                if file_name in all_sound_files:
                    wav_path = all_sound_files[file_name]
                if file_name in exts.lab_files:
                    lab_name = exts.lab_files[file_name]
                    transcription_path = os.path.join(root, lab_name)

                elif file_name in exts.textgrid_files:
                    tg_name = exts.textgrid_files[file_name]
                    transcription_path = os.path.join(root, tg_name)
                if wav_path is None:
                    continue
                self.job_queue.put((file_name, wav_path, transcription_path, relative_path))
                self.file_counts.increment()

        self.finished_adding.set()

class CorpusProcessWorker(threading.Thread):
    """
    Multiprocessing corpus loading worker

    Attributes
    ----------
    job_q: :class:`~multiprocessing.Queue`
        Job queue for files to process
    return_dict: dict
        Dictionary to catch errors
    return_q: :class:`~multiprocessing.Queue`
        Return queue for processed Files
    stopped: :class:`~threading.Event`
        Stop check for whether corpus loading should exit
    finished_adding: :class:`~threading.Event`
        Signal that the main thread has stopped adding new files to be processed
    """

    def __init__(
        self,
        name: int,
        job_q: Queue,
        return_q: Queue,
        stopped: threading.Event,
        finished_adding: threading.Event,
        speaker_characters: typing.Union[int, str],
        sample_rate: typing.Optional[int],
    ):
        super().__init__()
        self.name = str(name)
        self.job_q = job_q
        self.return_q = return_q
        self.stopped = stopped
        self.finished_adding = finished_adding
        self.finished_processing = threading.Event()
        self.speaker_characters = speaker_characters
        self.sample_rate = sample_rate

    def run(self) -> None:
        """
        Run the corpus loading job
        """
        while True:
            try:
                file_name, wav_path, text_path, relative_path = self.job_q.get(timeout=1)
            except Empty:
                if self.finished_adding.is_set():
                    break
                continue
            if self.stopped.is_set():
                continue
            try:
                file = FileData.parse_file(
                    file_name,
                    wav_path,
                    text_path,
                    relative_path,
                    self.speaker_characters,
                    self.sample_rate,
                )
                self.return_q.put(file)
            except TextParseError as e:
                self.return_q.put(("decode_error_files", e))
            except TextGridParseError as e:
                self.return_q.put(("textgrid_read_errors", e))
            except SoundFileError as e:
                self.return_q.put(("sound_file_errors", e))
            except Exception as e:
                self.stopped.set()
                self.return_q.put(("error", e))
        self.finished_processing.set()
        return

@dataclass
class NormalizeTextArguments(MfaArguments):
    """
    Arguments for :class:`~montreal_forced_aligner.corpus.multiprocessing.NormalizeTextFunction`

    """

    tokenizers: typing.Union[typing.Dict[int, SimpleTokenizer], Language]
    g2p_model: typing.Optional[G2PModel]

@dataclass
class ExportKaldiFilesArguments(MfaArguments):
    """
    Arguments for :class:`~montreal_forced_aligner.corpus.multiprocessing.NormalizeTextFunction`

    Parameters
    ----------

    """

    split_directory: Path

class NormalizeTextFunction(KaldiFunction):
    """
    Multiprocessing function for normalizing text.
    Parameters
    ----------
    args: :class:`~montreal_forced_aligner.corpus.multiprocessing.NormalizeTextArguments`
        Arguments for the function
    """

    def __init__(self, args: NormalizeTextArguments):
        super().__init__(args)
        self.tokenizers = args.tokenizers
        self.g2p_model = args.g2p_model

    def _run(self):
        """Run the function"""

        from montreal_forced_aligner.tokenization.simple import SimpleTokenizer

        with self.session() as session:
            dict_count = session.query(Dictionary).join(Dictionary.words).limit(1).count()
            if dict_count > 0 or isinstance(self.tokenizers, dict):
                dictionaries = session.query(Dictionary)
                for d in dictionaries:
                    if isinstance(self.tokenizers, dict):
                        tokenizer = self.tokenizers[d.id]
                    else:
                        tokenizer = self.tokenizers
                    simple_tokenization = isinstance(tokenizer, SimpleTokenizer)
                    if isinstance(tokenizer, Language):
                        from montreal_forced_aligner.tokenization.spacy import (
                            generate_language_tokenizer,
                        )

                        tokenizer = generate_language_tokenizer(tokenizer)

                    utterances = (
                        session.query(Utterance.id, Utterance.text)
                        .join(Utterance.speaker)
                        .filter(Utterance.text != "")
                        .filter(Utterance.job_id == self.job_name)
                        .filter(Speaker.dictionary_id == d.id)
                    )
                    for u_id, u_text in utterances:
                        if simple_tokenization:
                            normalized_text, normalized_character_text, oovs = tokenizer(u_text)
                            self.callback(
                                (
                                    {
                                        "id": u_id,
                                        "oovs": " ".join(sorted(oovs)),
                                        "normalized_text": normalized_text,
                                        "normalized_character_text": normalized_character_text,
                                    },
                                    d.id,
                                )
                            )
                        else:
                            normalized_text, pronunciation_form = tokenizer(u_text)
                            oovs = set()
                            self.callback(
                                (
                                    {
                                        "id": u_id,
                                        "oovs": " ".join(sorted(oovs)),
                                        "normalized_text": normalized_text,
                                        "normalized_character_text": pronunciation_form,
                                    },
                                    d.id,
                                )
                            )
            else:
                tokenizer = self.tokenizers
                utterances = (
                    session.query(Utterance.id, Utterance.text)
                    .filter(Utterance.text != "")
                    .filter(Utterance.job_id == self.job_name)
                )
                for u_id, u_text in utterances:
                    normalized_text, normalized_character_text, oovs = tokenizer(u_text)
                    self.callback(
                        (
                            {
                                "id": u_id,
                                "oovs": " ".join(sorted(oovs)),
                                "normalized_text": normalized_text,
                                "normalized_character_text": normalized_character_text,
                            },
                            None,
                        )
                    )

class ExportKaldiFilesFunction(KaldiFunction):
    """
    Multiprocessing function for normalizing text.
    Parameters
    ----------
    args: :class:`~montreal_forced_aligner.corpus.multiprocessing.NormalizeTextArguments`
        Arguments for the function
    """

    def __init__(self, args: ExportKaldiFilesArguments):
        super().__init__(args)
        self.split_directory = args.split_directory

    def output_to_directory(self, session) -> None:
        """
        Output job information to a directory

        Parameters
        ----------
        split_directory: str
            Directory to output to
        """
        job = (
            session.query(Job)
            .options(joinedload(Job.corpus, innerjoin=True), subqueryload(Job.dictionaries))
            .filter(Job.id == self.job_name)
            .first()
        )
        base_utterance_query = (
            session.query(sqlalchemy.func.count(Utterance.id))
            .filter(Utterance.job_id == job.id)
            .filter(Utterance.ignored == False)  # noqa
        )
        if job.corpus.current_subset:
            base_utterance_query = base_utterance_query.filter(Utterance.in_subset == True)  # noqa

        if not base_utterance_query.scalar():
            return
        if not job.has_dictionaries:
            utterances = (
                session.query(
                    Utterance.id,
                    Utterance.speaker_id,
                    Utterance.features,
                    Utterance.vad_ark,
                    Utterance.ivector_ark,
                    Speaker.cmvn,
                )
                .join(Utterance.speaker)
                .filter(Utterance.job_id == job.id)
                .filter(Utterance.ignored == False)  # noqa
                .order_by(Utterance.kaldi_id)
            )
            if job.corpus.current_subset:
                utterances = utterances.filter(Utterance.in_subset == True)  # noqa
            utt2spk_path = job.construct_path(self.split_directory, "utt2spk", "scp")
            feats_path = job.construct_path(self.split_directory, "feats", "scp")
            cmvns_path = job.construct_path(self.split_directory, "cmvn", "scp")
            spk2utt_path = job.construct_path(self.split_directory, "spk2utt", "scp")
            vad_path = job.construct_path(self.split_directory, "vad", "scp")
            ivectors_path = job.construct_path(self.split_directory, "ivectors", "scp")

            spk2utt = {}
            vad = {}
            ivectors = {}
            feats = {}
            cmvns = {}
            utt2spk = {}

            for (
                u_id,
                s_id,
                features,
                vad_ark,
                ivector_ark,
                cmvn,
            ) in utterances:
                utterance = str(u_id)
                speaker = str(s_id)
                utterance = f"{speaker}-{utterance}"
                if speaker not in spk2utt:
                    spk2utt[speaker] = []
                spk2utt[speaker].append(utterance)
                utt2spk[utterance] = speaker
                feats[utterance] = features
                if vad_ark:
                    vad[utterance] = vad_ark
                if ivector_ark:
                    ivectors[utterance] = ivector_ark
                cmvns[speaker] = cmvn
                self.callback(1)

            with mfa_open(spk2utt_path, "w") as f:
                for speaker, utts in sorted(spk2utt.items()):
                    utts = " ".join(sorted(utts))
                    f.write(f"{speaker} {utts}\n")

            with mfa_open(cmvns_path, "w") as f:
                for speaker, cmvn in sorted(cmvns.items()):
                    f.write(f"{speaker} {cmvn}\n")

            with mfa_open(utt2spk_path, "w") as f:
                for utt, spk in sorted(utt2spk.items()):
                    f.write(f"{utt} {spk}\n")

            with mfa_open(feats_path, "w") as f:
                for utt, feat in sorted(feats.items()):
                    f.write(f"{utt} {feat}\n")
            if vad:
                with mfa_open(vad_path, "w") as f:
                    for utt, ark in sorted(vad.items()):
                        f.write(f"{utt} {ark}\n")
            if ivectors:
                with mfa_open(ivectors_path, "w") as f:
                    for utt, ark in sorted(ivectors.items()):
                        f.write(f"{utt} {ark}\n")

        else:
            base_utterance_query = (
                session.query(
                    Utterance.id,
                    Utterance.speaker_id,
                    Utterance.features,
                    Speaker.cmvn,
                )
                .join(Utterance.speaker)
                .filter(Utterance.job_id == job.id)
                .filter(Utterance.ignored == False)  # noqa
                .order_by(Utterance.kaldi_id)
            )
            if job.corpus.current_subset:
                base_utterance_query = base_utterance_query.filter(
                    Utterance.in_subset == True  # noqa
                )
            utt2spk_paths = job.per_dictionary_utt2spk_scp_paths
            feats_paths = job.per_dictionary_feats_scp_paths
            cmvns_paths = job.per_dictionary_cmvn_scp_paths
            spk2utt_paths = job.per_dictionary_spk2utt_scp_paths
            for d in job.dictionaries:
                spk2utt = {}
                feats = {}
                cmvns = {}
                utt2spk = {}
                utterances = base_utterance_query.filter(Speaker.dictionary_id == d.id)
                for (
                    u_id,
                    s_id,
                    features,
                    cmvn,
                ) in utterances:
                    utterance = str(u_id)
                    speaker = str(s_id)
                    utterance = f"{speaker}-{utterance}"
                    if speaker not in spk2utt:
                        spk2utt[speaker] = []
                    spk2utt[speaker].append(utterance)
                    utt2spk[utterance] = speaker
                    feats[utterance] = features
                    cmvns[speaker] = cmvn
                    self.callback(1)

                with mfa_open(spk2utt_paths[d.id], "w") as f:
                    for speaker, utts in sorted(spk2utt.items()):
                        utts = " ".join(sorted(utts))
                        f.write(f"{speaker} {utts}\n")

                with mfa_open(cmvns_paths[d.id], "w") as f:
                    for speaker, cmvn in sorted(cmvns.items()):
                        f.write(f"{speaker} {cmvn}\n")

                with mfa_open(utt2spk_paths[d.id], "w") as f:
                    for utt, spk in sorted(utt2spk.items()):
                        f.write(f"{utt} {spk}\n")

                with mfa_open(feats_paths[d.id], "w") as f:
                    for utt, feat in sorted(feats.items()):
                        f.write(f"{utt} {feat}\n")

    def _run(self):
        """Run the function"""
        with self.session() as session:
            self.output_to_directory(session)


import cv2

video = cv2.VideoCapture(0)
# changed the code
while True:
    vid,frm = video.read()

    if vid==True:
        cv2.imshow('image',frm)
        if cv2.waitKey(1) & 0xff==ord('q'):
            break

video.release()
cv2.destroyAllWindows()


import cv2
import dlib

# Function to calculate the focal length of the camera
def FocalLength(measured_distance, real_width, width_in_ref_image):
    focal_length = (width_in_ref_image * measured_distance)/ real_width
    return focal_length

# Distance estimation function
def Distance_finder (Focal_Length, real_face_width, face_width_in_frame):
    distance = (real_face_width * Focal_Length)/face_width_in_frame
    return distance

# Face Detection by Haar-Cascade
face_detector = cv2.CascadeClassifier("haarcascade_frontalface_default.xml")

# For width of the face in frame
def face_data(image):
    face_width = 0
    gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    faces = face_detector.detectMultiScale(gray_image, 1.3, 5)
    for (x, y, h, w) in faces:
        cv2.rectangle(image, (x, y), (x+w, y+h), (222,141,123), 1)
        face_width = w
    return face_width

# For capturing distance from camera
def displaying_distance(cap):
    # video = cv2.VideoCapture(0)
    previous_distance = 30
    width = 14.3

    # To compare the distance we need another image
    img = cv2.imread("pic.jpg")
    face_width = face_data(img)
    Focal_length = FocalLength(previous_distance, width,face_width)
    print("Focal length : ", Focal_length)

    # k = True
    while True:
        ret, frame = cap.read()

        # while (k==True):

        #     k=False
        #     break

        if ret==True:
            face_width_in_frame = face_data(frame)
            if face_width_in_frame !=0:
                Distance = round(Distance_finder(Focal_length, width,face_width_in_frame),2)
                cv2.putText(frame, "Distance from Camera "+"{}".format(Distance)+"CM", (50,50), cv2.FONT_HERSHEY_COMPLEX,1, (123,246,123),3)

        cv2.imshow("frame", frame )
        if cv2.waitKey(1)==27:
            break

    cap.release()
    cv2.destroyAllWindows()
    # print("Distance captured : ", Distance)
    # return Distance


def countIdenticalShapes(matrix):
    seen_shapes = set()

    def normalize_shape(shape):
        # Find the top-leftmost cell in the shape
        min_row, min_col = min(shape, key=lambda x: (x[0], x[1]))

        # Normalize the shape by shifting all cells relative to the reference point
        normalized_shape_result = [(row - min_row, col - min_col) for row, col in shape]

        # Sort the normalized shape for consistency
        normalized_shape_result.sort()

        return normalized_shape_result

    def dfs(row, col, shape):
        # Base case: Out of bounds or not part of the shape
        if row < 0 or row >= len(matrix) or col < 0 or col >= len(matrix[0]) or matrix[row][col] == 0:
            return

        # Mark cell as visited
        matrix[row][col] = 0

        # Update the shape relative to a reference point
        shape.append((row, col))

        # Recursive DFS in all 4 directions
        dfs(row + 1, col, shape)
        dfs(row - 1, col, shape)
        dfs(row, col + 1, shape)
        dfs(row, col - 1, shape)

    for row in range(len(matrix)):
        for col in range(len(matrix[0])):
            if matrix[row][col] == 1:
                shape = []
                dfs(row, col, shape)

                # Normalize and store the shape
                normalized_shape = normalize_shape(shape)
                seen_shapes.add(tuple(normalized_shape))

    return len(seen_shapes)

def test_count_identical_shapes():
    # Test Example 1
    matrix1 = [
        [1, 1, 0, 0, 0],
        [1, 1, 0, 0, 0],
        [0, 0, 1, 0, 0],
        [0, 0, 0, 1, 1],
    ]
    result1 = countIdenticalShapes(matrix1)
    assert result1 == 3, f"Example 1 failed. Expected: 3, Got: {result1}"

    # Test Example 2
    matrix2 = [
        [1, 1, 0, 0, 1],
        [1, 1, 0, 0, 1],
        [0, 0, 1, 0, 0],
        [0, 0, 1, 1, 1],
    ]
    result2 = countIdenticalShapes(matrix2)
    assert result2 == 3, f"Example 2 failed. Expected: 3, Got: {result2}"

    # Test Example 3
    matrix3 = [
        [1, 0, 0, 0, 1],
        [0, 1, 0, 1, 0],
        [0, 0, 1, 0, 0],
        [1, 0, 1, 0, 1],
    ]
    result3 = countIdenticalShapes(matrix3)
    assert result3 == 2, f"Example 3 failed. Expected: 2, Got: {result3}"

    # Add more test cases as needed

    print("All test cases passed!")

# Call the test function
test_count_identical_shapes()