import torch
from gtts import gTTS

import os
import keyboard

from tqdm import tqdm
import random
import speech_recognition as sr

def recognize_speech_from_mic(recognizer, microphone):
    """Transcribe speech from recorded from `microphone`.

    Returns a dictionary with three keys:
    "success": a boolean indicating whether or not the API request was
               successful
    "error":   `None` if no error occured, otherwise a string containing
               an error message if the API could not be reached or
               speech was unrecognizable
    "transcription": `None` if speech could not be transcribed,
               otherwise a string containing the transcribed text
    """
    # check that recognizer and microphone arguments are appropriate type
    if not isinstance(recognizer, sr.Recognizer):
        raise TypeError("`recognizer` must be `Recognizer` instance")

    if not isinstance(microphone, sr.Microphone):
        raise TypeError("`microphone` must be `Microphone` instance")

    # adjust the recognizer sensitivity to ambient noise and record audio
    # from the microphone
    with microphone as source:
        print("Listening...")
        recognizer.adjust_for_ambient_noise(source)
        audio = recognizer.listen(source)
    print("cold mic...")

    # set up the response object
    response = {"success": True, "error": None, "transcription": None}

    # try recognizing the speech in the recording
    # if a RequestError or UnknownValueError exception is caught,
    #     update the response object accordingly
    try:
        response["transcription"] = recognizer.recognize_google(audio)
    except sr.RequestError:
        # API was unreachable or unresponsive
        response["success"] = False
        response["error"] = "API unavailable"
    except sr.UnknownValueError:
        # speech was unintelligible
        response["error"] = "Unable to recognize speech"

    return (
        response
        if response["success"]
        else recognize_speech_from_mic(recognizer, microphone)
    )

def generate(model, tokenizer, context):
    # tokenize question and text as a pair
    encodings = tokenizer.encode_plus(context)
    input_ids = encodings["input_ids"]
    attention_mask = encodings["attention_mask"]
    output = model.generate(
        inputs=torch.tensor([input_ids]),
        attention_mask=torch.tensor([attention_mask]),
        do_sample=True,
        num_beams=3,
        max_new_tokens=50,
        temperature=1.8,
        repetition_penalty=1.32,
    )
    return tokenizer.decode(output[0])

def speak(text):
    myobj = gTTS(text=text, lang="en", slow=False)
    myobj.save("output/speech.mp3")
    # Playing the converted file
    return os.system("mpg321 output/speech.mp3")

def sample_context(path):
    with open(path, "r") as contexts:
        lines = contexts.readlines()
        return random.sample(lines, 1)[0]

def main():
    # Model
    with open("models/generator.pth", "rb") as f:
        model = torch.load(f, map_location=torch.device('cpu'))

    # Tokenizer
    with open("models/tokenizer.pth", "rb") as f:
        tokenizer = torch.load(f, map_location=torch.device('cpu'))

    # create recognizer and mic instances
    recognizer = sr.Recognizer()
    microphone = sr.Microphone()

    while True:
        print("\n\n\n\n\n\n\n\n\n\n\n\nReady.")
        if keyboard.read_key() == "space":
            # context = recognize_speech_from_mic(recognizer, microphone)["transcription"]
            context = sample_context("data/context.txt")
            idx_start = len(context)

            # Create output file
            with open("output/crazy.txt", "w") as crazy:
                # Generate while reseeding 5 times
                for _ in tqdm(range(5)):
                    context = generate(model, tokenizer, str(context).strip())
                    context = context.replace("\n", "").replace("<|endoftext|>", "").replace("<pad>", "")
                # Write completed generation to file
                crazy.write(context)
                # Display response
                print(context[idx_start:])
                # Create and play an audio file of the output
                #  speak(context[idx_start:])

main()


import json
import plotly
import pandas as pd

from nltk.stem import WordNetLemmatizer
from nltk.tokenize import word_tokenize
from textblob import TextBlob
import numpy as np
from flask import Flask
from flask import render_template, request, jsonify
from plotly.graph_objs import Bar, Histogram
from sklearn.externals import joblib
from sqlalchemy import create_engine

ONE_LABEL_COL = "child_alone"

app = Flask(__name__)

def tokenize(text):
    tokens = word_tokenize(text)
    lemmatizer = WordNetLemmatizer()

    clean_tokens = []
    for tok in tokens:
        clean_tok = lemmatizer.lemmatize(tok).lower().strip()
        clean_tokens.append(clean_tok)

    return clean_tokens

# load data
engine = create_engine('sqlite:///../data/DisasterResponse.db')
df = pd.read_sql_table('tb_msg', engine)

# load model
model = joblib.load("../models/classifier.pkl")

# index webpage displays cool visuals and receives user input text for model
@app.route('/')
@app.route('/index')
def index():

    ## For genre analysis
    # extract data needed for visuals
    genre_counts = df.groupby('genre').count()['message']
    genre_names = list(genre_counts.index)
    pos_ratios = list((df[df.columns[4:]] > 0).mean(axis=1))
    cat_names = list(df.columns[4:])
    message_lengths = df.message.apply(lambda text: len(TextBlob(text).tokens)).values
    # create visuals
    graphs = [
        {
            'data': [
                Bar(
                    x=genre_names,
                    y=genre_counts,
                )
            ],

            'layout': {
                'title': 'Distribution of Message Genres',
                'yaxis': {
                    'title': "Count"
                },
                'xaxis': {
                    'title': "Genre"
                }
            }
        },
        {
            'data': [
                Bar(
                        x=cat_names,
                        y=pos_ratios
                )
            ],

            'layout': {
                'title': 'Distribution of Non-Zero labels in Each Category',
                'yaxis': {
                    'title': "Ratio of Positive Instances"
                },
                'xaxis': {
                    'title': "Category Name"
                }
            }
        },
        {
            'data': [
                Histogram(
                        x=message_lengths,
                        xbins=dict(start=np.min(message_lengths), size=0.8, end=np.max(message_lengths))
                    )
            ],

            'layout': {
                'title': 'Distribution of Message Length',
                'yaxis': {
                    'title': "Count"
                },
                'xaxis': {
                    'title': "Message Length"
                }
            }
        }
    ]

    # encode plotly graphs in JSON
    ids = ["graph-{}".format(i) for i, _ in enumerate(graphs)]
    graphJSON = json.dumps(graphs, cls=plotly.utils.PlotlyJSONEncoder)

    # render web page with plotly graphs
    return render_template('master.html', ids=ids, graphJSON=graphJSON)

# web page that handles user query and displays model results
@app.route('/go')
def go():
    # save user input in query
    query = request.args.get('query', '')

    # use model to predict classification for query
    classification_labels = model.predict([query])[0].tolist()
    one_label_idx = df.columns[4:].tolist().index(ONE_LABEL_COL)
    classification_labels.insert(one_label_idx, 0)
    classification_results = dict(zip(df.columns[4:], classification_labels))

    # This will render the go.html Please see that file.
    return render_template(
        'go.html',
        query=query,
        classification_result=classification_results
    )

def main():
    app.run(host='0.0.0.0', port=3001, debug=True)

if __name__ == '__main__':
    main()


import robin_stocks.robinhood as r
import json
import pandas as pd

# Reads data about the previous trades from a JSON file and prints it
def read_trade_history(file_name):
    print("read_trade_history()")
    with open(file_name) as json_file:
        data = json.load(json_file)
    for sell_date, event in data.items():
        print(sell_date + ": ")
        for symbol, dict in event.items():
            quantity, price, change, percent, purchase_date = str(int(float(dict.get("quantity")))), dict.get("price"), dict.get("equity_change"), dict.get("percent_change"), dict.get("purchase_date")
            print("\tSold " + quantity + " shares of "+ symbol + " at " + price + ", " + change + " (" +
                percent + "%) profit/loss, bought on " + purchase_date)

#  Writes data about a trade to a JSON file, containing the sell date, buy date, buy price sell price, etc.
def update_trade_history(symbols, holdings_data, file_name):
    print("update_trade_history()")

    with open(file_name) as json_file:
        data = json.load(json_file)
    current_time = str(pd.Timestamp("now"))
    data[current_time] = ({})
    for symbol in symbols:
        data[current_time].update({symbol: holdings_data[symbol]})
    with open(file_name, 'w') as outfile:
        json.dump(data, outfile)

# Returns the amount of money you've gained/lost through trading since the creation of your account, minus dividends
def get_total_gains_minus_dividends():
    print("get_total_gains_minus_dividends()")

    holding = []
    holding.append('temp')

    profileData = r.load_portfolio_profile()
    print(profileData)
    allTransactions = r.get_bank_transfers()
    deposits = sum(float(x['amount']) for x in allTransactions if (x['direction'] == 'deposit')) # and (x['state'] == 'completed'))
    withdrawals = sum(float(x['amount']) for x in allTransactions if (x['direction'] == 'withdraw') and (x['state'] == 'completed'))

    money_invested = deposits - withdrawals
    print(deposits)
    dividends = r.get_total_dividends()
    percentDividend = dividends/money_invested*100
    totalGainMinusDividends =float(profileData['extended_hours_equity'])-dividends-money_invested
    return totalGainMinusDividends


from datetime import datetime
import pytz as pytz
import boto3

boto3.setup_default_session(profile_name='s3-viewer')
s3 = boto3.client('s3')
response = s3.list_buckets()
bucket_names = [bucket['Name'] for bucket in response['Buckets']]

print('Select the bucket:')
for name in bucket_names:
    print(name)
selected_bucket = input('Enter bucket name: ')

response = s3.list_objects(Bucket=selected_bucket)
object_names = [obj['Key'] for obj in response['Contents']]

print('\nSelect the object:')
for name in object_names:
    print(name)
selected_object = input('\nEnter object key: ')

response = s3.list_object_versions(Bucket=selected_bucket, Prefix=selected_object)
versions = response['Versions']
versions.sort(key=lambda version: version['LastModified'])

print('\nObject have the next versions:')
for version in versions:
    print(version['LastModified'])
input_date = input("\nEnter the date to get the object in the next format: 'YYYY-MM-dd HH:mm:ss': ")
date = datetime.strptime(input_date, '%Y-%m-%d %H:%M:%S').replace(tzinfo=pytz.UTC)

selected_version = None
for version in versions:
    if date > version['LastModified']:
        selected_version = version

if selected_version:
    response = s3.get_object(Bucket=selected_bucket, Key=selected_object, VersionId=selected_version['VersionId'])
    print(response['Body'].read().decode('utf-8'))
else:
    print('Such version is not exist')


from PIL import Image
import numpy as np
import matplotlib.cm as cm
import math

ASPECT_RATIO = 1.0 / 1.0
FRAME_HEIGHT = 8.0
FRAME_WIDTH = FRAME_HEIGHT * ASPECT_RATIO

XRES = 1000
YRES = 1000

x_min = -FRAME_WIDTH/2
x_max = FRAME_WIDTH/2
y_min = -FRAME_HEIGHT/2
y_max = FRAME_HEIGHT/2

x_values = np.linspace(x_min, x_max, XRES+1)
y_values = np.linspace(y_min, y_max, YRES+1)

#print(x_values)
#print(y_values)

pixels = []

for i in range(len(y_values) - 1)[::-1]:
    pixels.append([])
    for j in range(len(x_values) - 1):
        x1, x2 = x_values[j:j + 2]
        y1, y2 = y_values[i:i + 2]
        x, y = (x1 + x2)/2, (y1 + y2)/2

        if x > 0:
            y = -y

        # if -2 < x < 2 and -2 < y < 2:
        #     pixel = [0, 0, 0]
        # else:
        #     pixel = [255, 255, 255]

        z = (x+y*1j)

        if z.real != 0:
            theta = 2*(math.atan(z.imag/z.real) + np.pi/2)
        else:
            theta = np.pi

        pixel = [x * 255 for x in cm.viridis(theta/(2*np.pi))]

        pixels[-1].append(pixel)

array = np.array(pixels, dtype=np.uint8)

new_image = Image.fromarray(array)
new_image.save("img/z_squared.png")
new_image.show()


"""Models and all supporting code"""

import cv2
import numpy as np
import utils

class OtsuThresholding(object):
    """Represents model for predicting i-contours given images and o-countours using Otsu's method"""

    def __init__(self, kernel_size=None):
        """Configures loader

        :param kernel_size: size of kernel for morphological closing, when `None` closing is not performed,
            defaults to `None`
        """

        self._kernel_size = kernel_size

    def predict(self, input):
        """Predicts i-contour

        :param input: tuple with images and o-contours
        :return: predicted i-contours
        """

        images, o_contours = input
        assert images.ndim == o_contours.ndim == 3
        assert images.shape == o_contours.shape
        assert o_contours.dtype == np.bool

        i_contours = []

        for image, o_contour in zip(images, o_contours):
            image = (image / image.max() * 255).astype(np.uint8)

            _, i_contour_vector = cv2.threshold(image[o_contour], 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
            i_contour_vector = i_contour_vector.squeeze()
            i_contour = np.copy(o_contour)
            i_contour[o_contour] = i_contour_vector

            if self._kernel_size:
                kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (self._kernel_size, self._kernel_size))
                i_contour = cv2.morphologyEx(i_contour.astype(np.uint8), cv2.MORPH_CLOSE, kernel).astype(np.bool)
                i_contour[np.logical_not(o_contour)] = False  # ignore everything outside o-contour

            i_contours.append(i_contour)

        i_contours = np.array(i_contours)

        return i_contours

    def score(self, input, i_contours):
        """Computes mean IOU on the given data and labels

        :param input: tuple with images and o-contours
        :param i_contours: i-contours labels
        :return: mean IOU
        """

        i_contours_pred = self.predict(input)

        return utils.iou(i_contours, i_contours_pred)


from django.shortcuts import render
from django.http import JsonResponse
import json
from django.shortcuts import get_object_or_404
from django.http import Http404
from django.views.decorators.csrf import csrf_exempt
from . import models

def index(request):
	ctx = {}
	return render(request, 'main/index.html', ctx)

def charts(request, onlyChart, sickness):
	ctx = {}
	aux = {}
	aux2 = {}
	if sickness == 'all':
		queryset = models.Case.objects.all()
	else:
		queryset = models.Case.objects.filter(sickness=sickness)
	for case in queryset:
		code = case.municipality.state.country.code
		year = case.date.year
		if aux.has_key(code) and aux[code].has_key(year):
			aux[code][year] += 1
		else:
			aux[code] = {year : 1}

		if aux2.has_key(year):
			aux2[year] += 1
		else:
			aux2[year] = 1

	world = []
	for year in aux2:
		world.append([year, aux2[year]])

	countries = {}
	data = []
	for code in aux:
		code_data = []
		count = 0
		for year in aux[code]:
			code_data.append([year, aux[code][year]])
			count += aux[code][year]
		data.append({'code3': code, 'value':count})
		countries[code]={
			'code3': code,
			'data': code_data
		}

	ctx['main_world'] = {
		'data': data,
		'all': world,
		'countries': countries
	}

	world = {}
	countries_data = {}
	for case in queryset:
		code = case.municipality.state.country.code
		year = case.date.year
		if countries_data.has_key(code) and countries_data[code].has_key(year):
			if case.age < 13:
				countries_data[code][year][0] += 1
			elif case.age < 19:
				countries_data[code][year][1] += 1
			elif case.age < 31:
				countries_data[code][year][2] += 1
			else:
				countries_data[code][year][3] += 1
		else:
			aux = [0, 0, 0, 0]
			if case.age < 13:
				aux[0] = 1
			elif case.age < 19:
				aux[1] = 1
			elif case.age < 31:
				aux[2] = 1
			else:
				aux[3] = 1
			countries_data[code] = {year: aux}

		if world.has_key(year):
			if case.age < 13:
				world[year][0] += 1
			elif case.age < 19:
				world[year][1] += 1
			elif case.age < 31:
				world[year][2] += 1
			else:
				world[year][3] += 1
		else:
			aux = [0, 0, 0, 0]
			if case.age < 13:
				aux[0] = 1
			elif case.age < 19:
				aux[1] = 1
			elif case.age < 31:
				aux[2] = 1
			else:
				aux[3] = 1
			world[year] = aux

	bar_chart_values = [[],[],[],[]]
	bar_chart_totals = [0, 0, 0, 0]
	for i in world.values():
		for j in range(4):
			bar_chart_values[j].append(i[j])
			bar_chart_totals[j] += i[j]

	countries_values = {}
	countries_totals = {}
	for code in countries_data:
		values = [[],[],[],[]]
		totals = [0, 0, 0, 0]
		for i in countries_data[code].values():
			for j in range(4):
				values[j].append(i[j])
				totals[j] += i[j]
		countries_values[code] = values
		countries_totals[code] = totals

	ctx['bar_chart'] = {
		'categories': world.keys(),
		'data': bar_chart_values,
		'totals': bar_chart_totals,
		'countries_values': countries_values,
		'countries_totals': countries_totals
	}

	aux = {}
	aux2={}
	for case in queryset.filter(recovered=False):
		code = case.municipality.state.country.code
		date = json.dumps(case.date.isoformat())
		if aux.has_key(code) and aux[code].has_key(date):
			aux[code][date].append(case.age)
		else:
			aux[code] = {date : [case.age]}

		if aux2.has_key(date):
			aux2[date].append(case.age)
		else:
			aux2[date] = [case.age]

	not_recovered_total = []
	for date in aux2:
		for age in aux2[date]:
			not_recovered_total.append([date, age])
	countries_values = {}
	countries_total = []
	'''for code in aux:
		code_data = []
		count = 0
		for year in aux[code]:
			code_data.append([date, aux[code][year]])
			count += aux[code][date]
		countries_total.append({'code3': code, 'value':count})
		countries_values[code]={
			'code3': code,
			'data': code_data
		}'''

	ctx['scatter_chart'] = {
		'total': not_recovered_total,
		'countries_values': countries_values,
		'countries_totals': countries_totals
	}

	if not onlyChart:
		return render(request, 'main/charts.html', ctx)
	else:
		return render(request, 'main/charts/main_chart.html', ctx)


import gym

from stable_baselines3.common.vec_env.dummy_vec_env import DummyVecEnv
from stable_baselines3 import PPO, SAC
from stable_baselines3.common.callbacks import EvalCallback
from stable_baselines3.common.utils import get_schedule_fn
from stable_baselines3.common.monitor import Monitor
from stable_baselines3.common.distributions import DiagGaussianDistribution

from src.environment_wrappers.env_wrappers import RewardWrapper, SkillWrapper, SkillWrapperFinetune
from src.environment_wrappers.tasks_wrappers import HalfCheetahTaskWrapper, WalkerTaskWrapper, AntTaskWrapper
from src.utils import best_skill
from src.mi_lower_bounds import mi_lower_bound
from src.models.models import Discriminator, SeparableCritic, ConcatCritic
from src.replayBuffers import DataBuffer
from src.callbacks.callbacks import DiscriminatorCallback, VideoRecorderCallback, EvaluationCallback, MI_EvalCallback
from src.config import conf

import torch
import torch.nn as nn
import torch.optim as opt
from torch.utils.tensorboard import SummaryWriter
import copy

import time

from collections import namedtuple

import matplotlib.pyplot as plt
import seaborn as sns
sns.set_theme(style="darkgrid")
sns.set(font_scale = conf.font_scale)

class DIAYN():
    '''
    An implementation of DIAYN with a model-free algorithm (PPO, SAC) as the optimization backbone
    '''
    def __init__(self, params, alg_params, discriminator_hyperparams, env="MountainCarContinuous-v0", alg="ppo", directory="./", seed=10, conf=None, timestamp=None, checkpoints=False, args=None, task=None, adapt_params=None, n_samples=None):
        # create the discirminator
        state_dim = gym.make(env).observation_space.shape[0]
        skill_dim = params['n_skills']
        hidden_dims = conf.num_layers_discriminator*[conf.layer_size_discriminator]
        latent_dim = conf.latent_size
        temperature = discriminator_hyperparams['temperature']
        dropout = discriminator_hyperparams['dropout']
        if discriminator_hyperparams['parametrization'] == "MLP":
            self.d = Discriminator(state_dim, hidden_dims, skill_dim, dropout=discriminator_hyperparams['dropout']).to(conf.device)
            self.d_adapt = Discriminator(state_dim, hidden_dims, skill_dim, dropout=discriminator_hyperparams['dropout']).to(conf.device)
        elif discriminator_hyperparams['parametrization'] == "Linear":
            self.d = nn.Linear(state_dim, skill_dim).to(conf.device)
        elif discriminator_hyperparams['parametrization'] == "Separable":
            # state_dim, skill_dim, hidden_dims, latent_dim, temperature=1, dropout=None)
            self.d = SeparableCritic(state_dim, skill_dim, hidden_dims, latent_dim, temperature, dropout).to(conf.device)
        elif discriminator_hyperparams['parametrization'] == "Concat":
            # state_dim, skill_dim, hidden_dims, temperature=1, dropout=None)
            self.d = ConcatCritic(state_dim, skill_dim, hidden_dims, temperature, dropout).to(conf.device)
        else:
            raise ValueError(f"{discriminator_hyperparams['parametrization']} is invalid parametrization")
        # Create the data buffer for the discriminator
        self.buffer = DataBuffer(params['buffer_size'], obs_shape=state_dim)

        # tensorboard summary writer
        self.sw = SummaryWriter(
            log_dir=directory, comment=f"{alg} Discriminator, env_name:{env}")

        # save some attributes
        self.alg = alg
        self.params = params
        self.alg_params = alg_params
        self.discriminator_hyperparams = discriminator_hyperparams
        self.env_name = env
        self.directory = directory
        self.seed = seed
        self.timestamp = timestamp
        self.conf = conf
        self.checkpoints = checkpoints
        self.args = args
        self.task = task
        self.parametrization = discriminator_hyperparams['parametrization']
        self.adapt_params = adapt_params
        self.n_samples = n_samples

    def pretrain(self):
        '''
        Pretraining phase with an intrinsic reward
        '''
        if self.alg == "ppo":
            env = DummyVecEnv([lambda: Monitor(RewardWrapper(SkillWrapper(gym.make(
                    self.env_name), self.params['n_skills'], max_steps=self.conf.max_steps), self.d, self.params['n_skills'], parametrization=self.parametrization),  self.directory)]*self.alg_params['n_actors'])

            # create the model with the speicifed hyperparameters
            model = PPO('MlpPolicy', env, verbose=1,
                        learning_rate=self.alg_params['learning_rate'],
                        n_steps=self.alg_params['n_steps'],
                        batch_size=self.alg_params['batch_size'],
                        n_epochs=self.alg_params['n_epochs'],
                        gamma=self.alg_params['gamma'],
                        gae_lambda=self.alg_params['gae_lambda'],
                        clip_range=self.alg_params['clip_range'],
                        policy_kwargs=dict(activation_fn=nn.ReLU,
                                           net_arch=[dict(pi=[self.conf.layer_size_policy, self.conf.layer_size_policy], vf=[self.conf.layer_size_value, self.conf.layer_size_value])]),
                        tensorboard_log=self.directory,
                        seed=self.seed
                        )

            discriminator_callback = DiscriminatorCallback(self.d, self.buffer, self.discriminator_hyperparams,
                                                           sw=self.sw, n_skills=self.params['n_skills'], min_buffer_size=self.params['min_train_size'], save_dir=self.directory, on_policy=True)

            # TODO: Replace this when you implement the other MI lower bounds with on policy algorithms
            eval_env = RewardWrapper(SkillWrapper(gym.make(
                self.env_name), self.params['n_skills'], ev=True), self.d, self.params['n_skills'], parametrization=self.parametrization)
            eval_env = Monitor(eval_env, f"{self.directory}/eval_results")
            eval_callback = EvalCallback(eval_env, best_model_save_path=self.directory,
                                         log_path=f"{self.directory}/eval_results", eval_freq=5000,
                                         deterministic=True, render=False, n_eval_episodes=self.conf.eval_runs)
            # create the callback list
            if self.checkpoints:
                # env_name, alg, discriminator, params, pm, n_samples=100)
                fineune_callback = EvaluationCallback(self.env_name, self.alg, self.d, self.adapt_params,self.params, self.parametrization, self.seed, self.directory, self.sw, n_samples=self.n_samples)
                callbacks = [discriminator_callback, eval_callback, fineune_callback]
            else:
                callbacks = [discriminator_callback, eval_callback]
            # train the agent
            model.learn(total_timesteps=self.params['pretrain_steps'], callback=callbacks, log_interval=3, tb_log_name="PPO Pretrain")
            # for testing
            # model.learn(total_timesteps=4500, callback=callbacks, log_interval=3, tb_log_name="PPO Pretrain")
        elif self.alg == "sac":
            env = DummyVecEnv([lambda: Monitor(RewardWrapper(SkillWrapper(gym.make(self.env_name), self.params['n_skills'], max_steps=self.conf.max_steps),
                                                             self.d, self.params['n_skills'], parametrization=self.parametrization), self.directory)])
            # create the model with the speicifed hyperparameters
            model = SAC('MlpPolicy', env, verbose=1,
                        learning_rate=self.alg_params['learning_rate'],
                        batch_size=self.alg_params['batch_size'],
                        gamma=self.alg_params['gamma'],
                        buffer_size=self.alg_params['buffer_size'],
                        tau=self.alg_params['tau'],
                        ent_coef=self.alg_params['ent_coef'],
                        gradient_steps=self.alg_params['gradient_steps'],
                        learning_starts=self.alg_params['learning_starts'],
                        policy_kwargs=dict(net_arch=dict(pi=[self.conf.layer_size_policy, self.conf.layer_size_policy], qf=[
                                           self.conf.layer_size_q, self.conf.layer_size_q]), n_critics=2),
                        tensorboard_log=self.directory,
                        seed=self.seed
                        )

            discriminator_callback = DiscriminatorCallback(self.d, None, self.discriminator_hyperparams, sw=self.sw,
                                                           n_skills=self.params['n_skills'], min_buffer_size=self.params['min_train_size'], save_dir=self.directory, on_policy=False)
            if self.parametrization in ["MLP", "Linear"]:
                eval_env = RewardWrapper(SkillWrapper(gym.make(
                    self.env_name), self.params['n_skills'], ev=True), self.d, self.params['n_skills'], parametrization=self.parametrization)
                eval_env = Monitor(eval_env,  f"{self.directory}/eval_results")

                eval_callback = EvalCallback(eval_env, best_model_save_path=self.directory,
                                             log_path=f"{self.directory}/eval_results", eval_freq=self.conf.eval_freq, deterministic=True, render=False)
            elif self.parametrization in ["Separable", "Concat"]:
                MI_estimator = namedtuple('MI_estimator', "estimator_func estimator_type log_baseline alpha_logit")
                mi_estimate = MI_estimator(mi_lower_bound, self.discriminator_hyperparams['lower_bound'], self.discriminator_hyperparams['log_baseline'], self.discriminator_hyperparams['alpha_logit'])
                # (env_name, discriminator, params, tb_sw, discriminator_hyperparams, mi_estimator, model_save_path, eval_freq=5000, verbose=0)
                eval_callback = MI_EvalCallback(self.env_name, self.d, self.params, self.sw, self.discriminator_hyperparams, mi_estimate, self.directory, eval_freq=self.conf.eval_freq*self.alg_params['n_actors'])

            # create the callback list
            if self.checkpoints:
                # env_name, alg, discriminator, params, pm, n_samples=100)
                fineune_callback = EvaluationCallback(self.env_name, self.alg, self.d, self.adapt_params,self.params, self.parametrization, self.seed, self.directory, self.sw, n_samples=self.n_samples)
                callbacks = [discriminator_callback, eval_callback, fineune_callback]
            else:
                callbacks = [discriminator_callback, eval_callback]

            # train the agent
            if self.parametrization in ["Separable", "Concat"]:
                model.learn(total_timesteps=self.params['pretrain_steps'], callback=callbacks, log_interval=3, tb_log_name="SAC Pretrain", d=self.d, mi_estimator=mi_estimate)
            elif self.parametrization in ["MLP", "Linear"]:
                model.learn(total_timesteps=self.params['pretrain_steps'], callback=callbacks, log_interval=3, tb_log_name="SAC Pretrain")
                 # for testing
                # model.learn(total_timesteps=2000, callback=callbacks, log_interval=3, tb_log_name="SAC Pretrain")
            else:
                raise ValueError(f"{discriminator_hyperparams['parametrization']} is invalid parametrization")
        if self.checkpoints:
            results = {
                'steps': fineune_callback.steps,
                'intr_rewards': fineune_callback.intr_rewards,
                'extr_rewards': fineune_callback.extr_rewards,
            }
            return model, self.d, results
        return model, self.d

    # finetune the pretrained policy on a specific task
    def finetune(self):
        '''
        Adapt the pretrained model with task reward using SAC
        '''
        # For the generalization experiment
        if self.task:
            # TODO: add other environments
            if self.env_name == "HalfCheetah-v2":
                env = HalfCheetahTaskWrapper(gym.make("HalfCheetah-v2"), task=self.task)
                env = DummyVecEnv([lambda: SkillWrapper(env, self.params['n_skills'], max_steps=self.conf.max_steps)])
            # WalkerTaskWrapper, AntTaskWrapper
            elif self.env_name == "Walker2d-v2":
                env = WalkerTaskWrapper(gym.make("Walker2d-v2"), task=self.task)
                env = DummyVecEnv([lambda: SkillWrapper(env, self.params['n_skills'], max_steps=self.conf.max_steps)])
            elif self.env_name == "Ant-v2":
                env = AntTaskWrapper(gym.make("Ant-v2"), task=self.task)
                env = DummyVecEnv([lambda: SkillWrapper(env, self.params['n_skills'], max_steps=self.conf.max_steps)])
        else:
            env = DummyVecEnv([lambda: SkillWrapper(gym.make(
                self.env_name), self.params['n_skills'], max_steps=self.conf.max_steps)])

        model_dir = self.directory + "/best_model"

        # Extract the model and extract the best skill
        if self.alg == "sac":
            model = SAC.load(model_dir, env=env, seed=self.seed)
        elif self.alg == "ppo":
            model = PPO.load(model_dir, env=env,  clip_range=get_schedule_fn(
                self.alg_params['clip_range']), seed=self.seed)

        best_skill_index = best_skill(
            model, self.env_name,  self.params['n_skills'])
        # best_skill_index = 0

        del model
        # define the environment for the adaptation model
        env, eval_env = self.adaptation_environment(best_skill_index)

        self.adaptation_model = SAC('MlpPolicy', env, verbose=1,
                    learning_rate=self.adapt_params['learning_rate'],
                    batch_size=self.adapt_params['batch_size'],
                    gamma=self.adapt_params['gamma'],
                    buffer_size=self.adapt_params['buffer_size'],
                    tau=self.adapt_params['tau'],
                    ent_coef=self.adapt_params['ent_coef'],
                    gradient_steps=self.adapt_params['gradient_steps'],
                    learning_starts=self.adapt_params['learning_starts'],
                    policy_kwargs=dict(net_arch=dict(pi=[self.conf.layer_size_policy, self.conf.layer_size_policy], qf=[
                                       self.conf.layer_size_q, self.conf.layer_size_q]), n_critics=2),
                    tensorboard_log=self.directory,
                    seed=self.seed
                    )

        eval_env = Monitor(eval_env, f"{self.directory}/finetune_eval_results")

        eval_callback = EvalCallback(eval_env, best_model_save_path=self.directory + f"/best_finetuned_model_skillIndex:{best_skill_index}",
                                    log_path=f"{self.directory}/finetune_eval_results", eval_freq=self.conf.eval_freq,
                                    deterministic=True, render=False)

        # if the model for pretraining is SAC just load the discriminator
        if self.alg == "sac":
            sac_model = SAC.load(model_dir, env=env, tensorboard_log=self.directory)
            # self.adaptation_model = sac_model
            self.adaptation_model.actor.latent_pi.load_state_dict(sac_model.actor.latent_pi.state_dict())
            self.adaptation_model.actor.mu.load_state_dict(sac_model.actor.mu.state_dict())
            self.adaptation_model.actor.log_std.load_state_dict(sac_model.actor.log_std.state_dict())
            self.adaptation_model.actor.optimizer = opt.Adam(self.adaptation_model.actor.parameters(), lr=self.adapt_params['learning_rate'])
            # # load the discriminator
            # self.d.layers[0] = nn.Linear(gym.make(self.env_name).observation_space.shape[0] + self.params['n_skills']  + gym.make(self.env_name).action_space.shape[0], self.conf.layer_size_discriminator)
            # self.d.layers[-1] = nn.Linear(self.conf.layer_size_discriminator, 1)
            # seq = nn.Sequential(*self.d.layers)
            # # # print(d)
            # self.d.eval()
            self.adaptation_model.critic.qf0.load_state_dict(sac_model.critic.qf0.state_dict())
            self.adaptation_model.critic.qf1.load_state_dict(sac_model.critic.qf1.state_dict())
            self.adaptation_model.critic_target.load_state_dict(self.adaptation_model.critic.state_dict())
            self.adaptation_model.critic.optimizer = opt.Adam(self.adaptation_model.critic.parameters(), lr=self.adapt_params['learning_rate'])
            self.adaptation_model.learn(total_timesteps=self.params['finetune_steps'],
                        callback=eval_callback, tb_log_name="SAC_FineTune", d=None, mi_estimator=None)

        # if the model for the prratrining is PPO load the discrimunator and actor from ppo into sac models
        elif self.alg == "ppo":
            self.ppo_model = PPO.load(model_dir, env=env, tensorboard_log=self.directory,
                            clip_range=get_schedule_fn(self.alg_params['clip_range']))
            # adaptation_model.actor.action_dist = DiagGaussianDistribution(env.action_space.shape[0])
            # load the policy ppo
            ppo_actor = self.ppo_model.policy
            self.load_state_dicts(ppo_actor)
            self.adaptation_model.learn(total_timesteps=self.params['finetune_steps'],
                        callback=eval_callback, tb_log_name="PPO_FineTune", d=None, mi_estimator=None)
        return self.adaptation_model, best_skill_index

    def load_state_dicts(self, ppo_actor):
        '''
        load the pretrained model parameters into the adaptation model
        '''
        # initlialize the adaptation policy with the pretrained model
        self.adaptation_model.actor.latent_pi.load_state_dict(ppo_actor.mlp_extractor.policy_net.state_dict())
        self.adaptation_model.actor.mu.load_state_dict(ppo_actor.action_net.state_dict())
        self.adaptation_model.actor.log_std.load_state_dict(nn.Linear(in_features=self.conf.layer_size_policy, out_features=gym.make(self.env_name).action_space.shape[0], bias=True).state_dict())
        self.adaptation_model.actor.optimizer = opt.Adam(self.adaptation_model.actor.parameters(), lr=self.adapt_params['learning_rate'])
        layers = [nn.Linear(gym.make(self.env_name).observation_space.shape[0] + self.params['n_skills']  + gym.make(self.env_name).action_space.shape[0], self.conf.layer_size_discriminator)]
        for l in range(len(self.ppo_model.policy.mlp_extractor.value_net)):
            if l != 0:
                layers.append(self.ppo_model.policy.mlp_extractor.value_net[l])
        layers.append(nn.Linear(self.conf.layer_size_discriminator, 1))
        seq = nn.Sequential(*layers)
        print(seq)
        # input()
        self.adaptation_model.critic.qf0.load_state_dict(seq.state_dict())
        self.adaptation_model.critic.qf1.load_state_dict(seq.state_dict())
        self.adaptation_model.critic_target.load_state_dict(self.adaptation_model.critic.state_dict())
        self.adaptation_model.critic.optimizer = opt.Adam(self.adaptation_model.critic.parameters(), lr=self.adapt_params['learning_rate'])

    def adaptation_environment(self, best_skill_index):
        '''
        Adaptation environment according to the task reward
        '''
        if self.task:
            # print(f"There is a task which is {self.task}")
            # input()
            if self.env_name == "HalfCheetah-v2":
                # print(f"environment: {self.env_name}")
                # input()
                env = HalfCheetahTaskWrapper(gym.make("HalfCheetah-v2"), task=self.task)
                env = DummyVecEnv([lambda: SkillWrapperFinetune(Monitor(env,  f"{self.directory}/finetune_train_results"), self.params['n_skills'], max_steps=gym.make(self.env_name)._max_episode_steps, skill=best_skill_index)])
                eval_env = HalfCheetahTaskWrapper(gym.make(self.env_name), task=self.task)
                eval_env = SkillWrapperFinetune(eval_env, self.params['n_skills'], max_steps=gym.make(self.env_name)._max_episode_steps, r_seed=None, skill=best_skill_index)

            # # WalkerTaskWrapper, AntTaskWrapper
            elif self.env_name == "Walker2d-v2":
                # print(f"environment: {self.env_name}")
                # input()
                env = WalkerTaskWrapper(gym.make("Walker2d-v2"), task=self.task)
                env = DummyVecEnv([lambda: SkillWrapperFinetune(Monitor(env,  f"{self.directory}/finetune_train_results"), self.params['n_skills'], max_steps=gym.make(self.env_name)._max_episode_steps, skill=best_skill_index)])
                eval_env = WalkerTaskWrapper(gym.make(self.env_name), task=self.task)
                eval_env = SkillWrapperFinetune(eval_env, self.params['n_skills'], max_steps=gym.make(self.env_name)._max_episode_steps, r_seed=None, skill=best_skill_index)

            elif self.env_name == "Ant-v2":
                # print(f"environment: {self.env_name}")
                # input()
                env = AntTaskWrapper(gym.make("Ant-v2"), task=self.task)
                env = DummyVecEnv([lambda: SkillWrapperFinetune(Monitor(env,  f"{self.directory}/finetune_train_results"), self.params['n_skills'], max_steps=gym.make(self.env_name)._max_episode_steps, skill=best_skill_index)])
                eval_env = AntTaskWrapper(gym.make(self.env_name), task=self.task)
                eval_env = SkillWrapperFinetune(eval_env, self.params['n_skills'], max_steps=gym.make(self.env_name)._max_episode_steps, r_seed=None, skill=best_skill_index)

        else:
            # print(f"Just ")
            env = DummyVecEnv([lambda: SkillWrapperFinetune(Monitor(gym.make(
    self.env_name),  f"{self.directory}/finetune_train_results"), self.params['n_skills'], r_seed=None,max_steps=gym.make(self.env_name)._max_episode_steps, skill=best_skill_index)])
            eval_env = SkillWrapperFinetune(gym.make(
            self.env_name), self.params['n_skills'], max_steps=gym.make(self.env_name)._max_episode_steps, r_seed=None, skill=best_skill_index)

        return env, eval_env


import os
import mock
import json
import pika
import Queue
import logging
import greenlet
import unittest
import threading
from rackattack.tcp import publish
from rackattack.tcp import subscribe
from rackattack.tests import mock_pika
from rackattack.tests import one_threaded_publish

handler = logging.StreamHandler()
handler.setLevel(logging.INFO)
logging.getLogger().addHandler(handler)
logging.getLogger().setLevel(logging.INFO)

class Host:
    def id(self):
        return "myID"

class FakeHostStateMachine:
    def hostImplementation(self):
        return Host()

class Test(unittest.TestCase):
    def setUp(self):
        mock_pika.enableMockedPika(modules=[publish])
        self.tested = one_threaded_publish.OneThreadedPublish(mock_pika.DEFAULT_AMQP_URL)
        self.consumer = mock_pika.getBlockingConnectionToFakeBroker().channel()

    def tearDown(self):
        mock_pika.disableMockedPika(modules=[publish])

    def test_ThreadStarted(self):
        self.tested.threadStartMock.assert_called_once_with(self.tested)

    def test_BadURL(self):
        killMock = mock.Mock()
        origKill = os.kill
        try:
            os.kill = killMock
            publishInstance = greenlet.greenlet(
                one_threaded_publish.OneThreadedPublish('invalid amqp url').run)
            publishInstance.switch()
        finally:
            os.kill = origKill
        self.assertEquals(killMock.call_count, 1)

    def test_AllocationProviderMessage(self):
        self.tested.continueWithServer()
        allocationID = 2
        message = 'alpha bravo tango'
        self.tested.allocationProviderMessage(allocationID, message)
        self.tested.continueWithServer()
        expectedMessage = dict(event='providerMessage', allocationID=allocationID, message=message)
        expectedExchange = publish.PublishSpooler.allocationExchange(allocationID)
        actualMessage = self._consume(expectedExchange)
        self.assertEquals(actualMessage, expectedMessage)

    def test_AllocationRequested(self):
        self.tested.continueWithServer()
        allocationID = 1
        fields = dict(requirements=dict(theseAre="myRequirements"),
                      allocationInfo="whatACoolAllocation")
        self.tested.allocationRequested(**fields)
        self.tested.continueWithServer()
        expectedMessage = dict(fields, event='requested')
        expectedExchange = publish.Publish.ALL_HOSTS_ALLOCATIONS_EXCHANGE_NAME
        actualMessage = self._consume(expectedExchange)
        self.assertEquals(actualMessage, expectedMessage)

    def test_AllocationRejected(self):
        self.tested.continueWithServer()
        allocationID = 1
        self.tested.allocationRejected(reason="No Resources")
        self.tested.continueWithServer()
        expectedMessage = dict(event='rejected', reason="No Resources")
        expectedExchange = publish.Publish.ALL_HOSTS_ALLOCATIONS_EXCHANGE_NAME
        actualMessage = self._consume(expectedExchange)
        self.assertEquals(actualMessage, expectedMessage)

    def test_AllocationCreated(self):
        self.tested.continueWithServer()
        allocationID = 1
        hostStateMachine = FakeHostStateMachine()
        allocated = {"node0": hostStateMachine}
        self.tested.allocationCreated(allocationID=allocationID, allocated=allocated)
        self.tested.continueWithServer()
        expectedAllocated = {"node0": hostStateMachine.hostImplementation().id()}
        expectedMessage = dict(event='created', allocated=expectedAllocated, allocationID=allocationID)
        expectedExchange = publish.Publish.ALL_HOSTS_ALLOCATIONS_EXCHANGE_NAME
        actualMessage = self._consume(expectedExchange)
        self.assertEquals(actualMessage, expectedMessage)

    def test_AllocationDone(self):
        self.tested.continueWithServer()
        allocationID = 1
        self.tested.allocationDone(allocationID)
        self.tested.continueWithServer()
        expectedExchange = publish.PublishSpooler.allocationExchange(allocationID)
        message = self._consume(expectedExchange)
        self.assertEquals(message["event"], "changedState")
        expectedMessage = dict(event="done", allocationID=allocationID)
        expectedExchange = publish.Publish.ALL_HOSTS_ALLOCATIONS_EXCHANGE_NAME
        actualMessage = self._consume(expectedExchange)
        self.assertEquals(actualMessage, expectedMessage)

    def test_AllocationDied_NoWithdrawl(self):
        self.tested.continueWithServer()
        allocationID = 1
        client = self.tested.allocationDied(allocationID=allocationID, reason="freed", message="'sup")
        self.tested.continueWithServer()
        expectedExchange = publish.PublishSpooler.allocationExchange(allocationID)
        message = self._consume(expectedExchange)
        self.assertEquals(message["event"], "changedState")
        expectedMessage = dict(event="dead", allocationID=allocationID, reason="freed", moreInfo="'sup")
        expectedExchange = publish.Publish.ALL_HOSTS_ALLOCATIONS_EXCHANGE_NAME
        actualMessage = self._consume(expectedExchange)
        self.assertEquals(actualMessage, expectedMessage)

    def test_AllocationDied_Withdrawn(self):
        self.tested.continueWithServer()
        allocationID = 1
        self.tested.allocationDied(allocationID=allocationID, reason="withdrawn", message="'sup")
        self.tested.continueWithServer()
        expectedExchange = publish.PublishSpooler.allocationExchange(allocationID)
        message = self._consume(expectedExchange)
        self.assertEquals(message["event"], "withdrawn")
        expectedMessage = dict(event="dead", allocationID=allocationID, reason="withdrawn", moreInfo="'sup")
        expectedExchange = publish.Publish.ALL_HOSTS_ALLOCATIONS_EXCHANGE_NAME
        actualMessage = self._consume(expectedExchange)
        self.assertEquals(actualMessage, expectedMessage)

    def test_CleanupAllocationPublishResources(self):
        self.tested.continueWithServer()
        allocationID = 1
        self.tested.allocationDone(allocationID)
        self.tested.continueWithServer()
        allocationExchange = publish.PublishSpooler.allocationExchange(allocationID)
        self._consume(allocationExchange)
        expectedExchanges = ['',
                             allocationExchange,
                             publish.Publish.ALL_HOSTS_ALLOCATIONS_EXCHANGE_NAME]
        for expectedExchange in expectedExchanges:
            self.assertIn(expectedExchange, self.consumer.exchanges)
        self.assertIn(expectedExchange, self.consumer.exchanges)
        self.tested.cleanupAllocationPublishResources(allocationID)
        self.tested.continueWithServer()
        expectedExchanges = ['', publish.Publish.ALL_HOSTS_ALLOCATIONS_EXCHANGE_NAME]
        self.assertEquals(self.consumer.exchanges.keys(), expectedExchanges)

    def test_TryCleaningUpResourcesForANonExistingAllocationDoesNotCrashServer(self):
        self.tested.continueWithServer()
        allocationID = 1
        self.tested.cleanupAllocationPublishResources(allocationID)
        self.tested.continueWithServer()

    def test_CleaningUpResourcesForADeadAllocationDoesNotCrashServer(self):
        self.tested.continueWithServer()
        allocationID = 1
        self.tested.allocationDone(allocationID)
        self.tested.continueWithServer()
        allocationExchange = publish.PublishSpooler.allocationExchange(allocationID)
        self._consume(allocationExchange)
        expectedExchanges = ['',
                             allocationExchange,
                             publish.Publish.ALL_HOSTS_ALLOCATIONS_EXCHANGE_NAME]
        for expectedExchange in expectedExchanges:
            self.assertIn(expectedExchange, self.consumer.exchanges)
        self.assertIn(expectedExchange, self.consumer.exchanges)
        self.tested.cleanupAllocationPublishResources(allocationID)
        self.tested.continueWithServer()
        expectedExchanges.remove(allocationExchange)
        self.assertEquals(self.consumer.exchanges.keys(), expectedExchanges)
        self.tested.cleanupAllocationPublishResources(allocationID)
        self.tested.continueWithServer()
        self.assertEquals(self.consumer.exchanges.keys(), expectedExchanges)

    def _consume(self, exchange):
        message = self.consumer.basic_consume(exchange=exchange)
        return json.loads(message)

if __name__ == '__main__':
    unittest.main()


import os
import csv
from utilities.utilities import *
from utilities.katuyou import Katuyou
from collections import defaultdict

class EmotionClass:
    def __init__(self, feeling_folder_path, line_file_path):
        self.feeling_foloder_path = feeling_folder_path
        self.line_file_path = line_file_path
        files = os.listdir(feeling_folder_path)
        self.feeling_dict = defaultdict(list) #key is file name, value is feelings
        self.line_dict = defaultdict(list) #key is file name, value is line
        self.katuyou = Katuyou('./utilities/KatuyouCSVs')
        for file in files:
            name = file.split('.')[0]
            with open(os.path.join(self.feeling_foloder_path, file), 'r', encoding="utf-8") as f:
                feelings = f.readlines()
            for feeling in feelings:
                feeling = feeling.rstrip('\n')
                if feeling.startswith('#') or feeling.startswith('-'):
                    pass
                else:
                    self.feeling_dict[name].append(feeling)

        with open(self.line_file_path, 'r', encoding='utf-8') as f:
            lines = f.readlines()
            reader = csv.reader(lines)
            for row in reader:
                assert row[0] in self.feeling_dict.keys(), 'Not found file name at {}'.format(row[0])
                self.line_dict[row[0]] = row[1:]

    def current2past(self, current):
        escape_words = ['さすが', '流石', '立派', '知的', '頭', 'どうして', '素敵', '大丈夫', 'お疲れ様']
        for word in escape_words:
            if word in current:
                return current
        if 'がない' in current:
            current = current.replace('がない', 'がなかった')
        elif 'が無い' in current:
            current = current.replace('が無い', 'が無かった')
        elif 'なんだ' in current:
            current = current.replace('なんだ', 'だったんだ')
        elif 'いいん' in current:
            current = current.replace('いいん', 'よかったん')
        elif 'いん' in current:
            current = current.replace('いん', 'かったん')
        elif 'いね' in current:
            current = current.replace('いね', 'かったね')
        else:
            current = self._verb_cur2past(current)
        return current

    def _verb_cur2past(self, line): #動詞のやつを過去形に変換。
        tokens, positions = mecab_parse(line)
        res_line = ''
        for i, token, position in zip(range(len(tokens)-1, -1, -1), tokens[::-1], positions[::-1]):
            if position[0] == '助動詞' and i -1 >= 0 and (positions[i-1][0] not in ['動詞', '形容詞'] and tokens[i-1] not in ['ん']):
                token = self.katuyou.convert_base2ta(token)
                res_line = ''.join(tokens[:i]) + token  + 'たんだ' + res_line
                break
            elif position[0] in ['形容詞', '動詞']:
                token = self.katuyou.convert_base2ta(token)
                res_line = ''.join(tokens[:i]) + token + 'た' + res_line
                break
            else:
                res_line = token + res_line
        return res_line

    def Emotion(self, phrases):
        statement = ''
        apply = False
        if len(phrases) > 2:
            tokens = phrases[-3][0] + phrases[-2][0] + phrases[-1][0]
            positions = phrases[-3][1] + phrases[-2][1] + phrases[-1][1]
        elif len(phrases) == 2:
            tokens = phrases[-2][0] + phrases[-1][0]
            positions = phrases[-2][1] + phrases[-1][1]
        else:
            tokens, positions = phrases[-1]

        for i, token, position in zip(range(len(tokens)-1, -1, -1), tokens[::-1], positions[::-1]):
            if position[-1] in self.line_dict.keys():
                flag = get_flags(tokens[i:], positions[i:])
                if not flag['nai'] and not flag['reru']:
                    apply = True
                    statement = random.choice(self.line_dict[position[-1]])
                    if flag['ta']:
                        statement = self.current2past(statement)
                else:
                    pass
                break
        return statement, apply

    def passive_checker(self, tokens, positions):
        for i, token, position in zip(range(len(tokens)), tokens, positions):
            if position[0] == '動詞' and i + 2 < len(positions) and tokens[i + 1] in ['て', 'で'] \
                    and positions[i + 2][-3] in ['くださる', 'もらえる', 'くれる', 'もらう', '下さる']:
                return True
            elif position[0] == '動詞' and i + 1 < len(positions) and positions[i + 1][-3] in ['れる', 'られる', 'せる']:
                return True
            elif token in ['を'] and i + 1 < len(positions) and positions[i + 1][-3] in ['受ける', '迎える']:
                return True
            elif position[0] == '名詞' and position[1] not in ['非自立', '接尾']:
                return False
            return False

if __name__=='__main__':
    emotion = EmotionClass('utilities/FEELINGS', 'utilities/feelings2line.csv')
    from Preprocess import Preprocess
    sentences = read_lines()
    sentences = ['好きな男だった。']
    for sentence in sentences:
        sentence, apply, subject, conjunction, phrases, tokens, positions = Preprocess(sentence)
        print(phrases)
        print(emotion.Emotion(phrases))


import pygame

from player import Player

class GameScreen(object):

    def __init__(self, w, h, tilesize):
        pygame.init()

        #initialize all screen variables

        self.width  = w
        self.height = h
        self.scene   = pygame.display.set_mode( (w,h) )
        self.caption = pygame.display.set_caption("This is Game Title")
        self.icon    = pygame.image.load("assets/icon.png")
        #actually displays the icon
        pygame.display.set_icon(self.icon)

        #tilesize setup
        self.tileSize = tilesize
        self.gridColor = (130,98,22)

        # any game sound here.
        self.track   = pygame.mixer.music.load("assets/levelMusic.wav")
        self.m_loop  = pygame.mixer.music.play(-1)
        self.m_volum = pygame.mixer.music.set_volume(0)

        # also include background image

        self.bgc     = (51,51,51)

        #define player
        self.player =  Player(self, 5, 8)

        self.water = pygame.image.load("assets/water.png")
        self.grass = pygame.image.load("assets/grass.png")
        self.street = pygame.image.load("assets/road.png")
        self.CL    = pygame.image.load("assets/car_l.png")
        self.CR    = pygame.image.load("assets/car_r.png")
        self.LL    = pygame.image.load("assets/log_l.png")
        self.LM    = pygame.image.load("assets/log_m.png")
        self.LR    = pygame.image.load("assets/log_r.png")

        #roads
        self.roads = []
        self.roads.append(["","","CL","CR","","","","CL","CR",""])
        self.roads.append(["CR","","","","","","","","","CL"])
        self.roads.append(["","","","","","","CL","CR","",""])
        self.roads.append(["CL","CR","","","","","","","",""])
        self.roads.append(["","","CL","CR","","","","CL","CR",""])

        #logs
        self.logs = []
        self.logs.append(["LL","LM","LR","","","","","","",""])
        self.logs.append(["LR","","","","","","","","","LL"])
        self.logs.append(["","","LL","LM","LR","","","","",""])
        self.logs.append(["","LL","LR","","","","","LL","LR",""])
        self.logs.append(["","","","","LL","LM","LR","","",""])
        #self.object = ()

        self.clock = pygame.time.Clock()
        #self.timer = pygame.time.get_ticks()

        self.elapsedTime = 0

    def draw_grid(self):
        for x in range(0, self.width, self.tileSize):
            pygame.draw.line(self.scene, self.gridColor, (x, 0), (x, self.height) )
        for y in range(0, self.height, self.tileSize):
            pygame.draw.line(self.scene, self.gridColor, (0, y), (self.width, y) )

    def setBackground(self):
        self.scene.fill(self.bgc)
        self.draw_grid()

    def display_board(self):

        for i in range(10):
            coords = [i * self.tileSize, 0 * self.tileSize ]
            self.scene.blit(self.grass, ( coords[0] , coords[1] ) )

        for i in range(1,6):
            for u in range(10):
                coords = [u * self.tileSize, i * self.tileSize ]
                self.scene.blit(self.water, (coords[0], coords[1] ) )

        for i in range(10):
            coords = [i * self.tileSize, 6 * self.tileSize ]
            self.scene.blit(self.grass, ( coords[0] , coords[1] ) )

        for i in range(7,12):
            for u in range(10):
                coords = [u * self.tileSize, i * self.tileSize ]
                self.scene.blit(self.street, (coords[0], coords[1] ) )

        for i in range(10):
            coords = [i * self.tileSize, 12 * self.tileSize ]
            self.scene.blit(self.grass, ( coords[0] , coords[1] ) )

    def advance_rows(self):
        for i in range(len(self.roads)):
            row = self.roads[i]
            end = row.pop()
            row.insert(0, end)
            self.roads[i] = row
        for i in range(len(self.logs)):
            row = self.logs[i]
            end = row.pop()
            row.insert(0, end)
            self.logs[i] = row

    def display_objects(self):
        for i in range(len(self.roads)):
            row = self.roads[i]
            for u in range(len(row)):
                if(row[u] != ''):
                    if(row[u] == "CL"):
                        #print('-')
                        self.scene.blit(self.CL, (u * self.tileSize, (i+7) * self.tileSize) )
                    if(row[u] == "CR"):
                        #print('()')
                        self.scene.blit(self.CR, (u * self.tileSize, (i+7) * self.tileSize) )

        for i in range(len(self.logs)):
            row = self.logs[i]
            for u in range(len(row)):
                if(row[u] != ''):
                    if(row[u] == "LL"):
                        #print('-')
                        self.scene.blit(self.LL, (u * self.tileSize, (i+1) * self.tileSize) )
                    if(row[u] == "LM"):
                        #print('()')
                        self.scene.blit(self.LM, (u * self.tileSize, (i+1) * self.tileSize) )
                    if(row[u] == "LR"):
                        #print('()')
                        self.scene.blit(self.LR, (u * self.tileSize, (i+1) * self.tileSize) )

    def update(self):
        keepGoing = True

        self.setBackground()
        self.display_board()
        #self.draw_grid()

        dt = self.clock.tick()
        self.elapsedTime += dt

        if(self.elapsedTime > 1000):
            self.elapsedTime = 0
            self.advance_rows()

        self.display_objects()
        self.player.update()

        pygame.display.update()

        return keepGoing


from selenium.webdriver.chrome.service import Service

"""
@package base

WebDriver Factory class implementation
It creates a webdriver instance based on browser configurations
"""
from selenium import webdriver

class WebDriverFactory():

    def __init__(self, browser):
        """
        Inits WebDriverFactory class

        Returns:
            None
        """
        self.browser = browser

    def get_web_driver_instance(self):
        """
        Get WebDriver Instance based on the browser configuration

        Returns:
            'WebDriver Instance'
        """
        base_url = "https://www.hudl.com/login"
        if self.browser == "chrome":
            service = Service("./drivers/chromedriver.exe")
            driver = webdriver.Chrome(service=service)
        else:
            service = Service("./drivers/geckodriver.exe")
            driver = webdriver.Firefox(service=service)
        # Setting Driver Implicit Time out for An Element
        driver.implicitly_wait(3)
        # Maximize the window
        driver.maximize_window()
        # Loading browser with App URL
        driver.get(base_url)
        return driver


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri May 12 01:26:44 2023

@author: mingjunsun
"""

import pandas as pd
import numpy as np
import os
import torch
import torch.nn as nn
import torch.nn.functional as F
import random
import matplotlib.pyplot as plt
#from livelossplot import PlotLosses
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error

from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error, r2_score

#set working directory
os.chdir('/Users/mingjunsun/Library/CloudStorage/Dropbox/23 Summer/Data/')

data_original = pd.read_csv("Characteristics/usa.csv")

data = data_original
data_expl = data.head(105)

#exclude data before 1991-12-31
data = data[~(data['date'] < 19920000)]

#exclude observations with missing me in month t and missing return in month t+1
data = data.dropna(subset=['me', 'ret_exc_lead1m'])

#exclude nano caps
data = data.loc[data['size_grp'] != 'nano']

#delete observation with more than 5 out of the 21 characteristics missing
cols = ["be_me", "ret_12_1", "market_equity", "ret_1_0", "rvol_252d", "beta_252d", "qmj_safety", "rmax1_21d", "chcsho_12m","ni_me", "eq_dur", "ret_60_12", "ope_be", "gp_at", "ebit_sale", "at_gr1", "sale_gr1", "at_be","cash_at", "age", "z_score"]
data["missing_num"] = data[cols].isna().sum(1)
data = data.loc[data['missing_num'] <= 5]

#impute the missing characteristics by replacing them with the cross-sectional median
for i in cols:
    data[i] = data[i].astype(float)
    data[i] = data[i].fillna(data.groupby('date')[i].transform('median'))

#check missing values of the 21 characteristics
#randomlist = random.sample(range(1, 310000), 150)
#data_expl = data.iloc[randomlist]
#cols1 = ["missing_num", "id", "date","be_me", "ret_12_1", "market_equity", "ret_1_0", "rvol_252d", "beta_252d", "qmj_safety", "rmax1_21d", "chcsho_12m","ni_me", "eq_dur", "ret_60_12", "ope_be", "gp_at", "ebit_sale", "at_gr1", "sale_gr1", "at_be","cash_at", "age", "z_score"]
#data_expl = data_expl[cols1]

#predict next month's excess return over the training period

#OLS pytorch
cols1 = ["date","ret_exc_lead1m", "be_me", "ret_12_1", "market_equity", "ret_1_0", "rvol_252d", "beta_252d", "qmj_safety", "rmax1_21d", "chcsho_12m","ni_me", "eq_dur", "ret_60_12", "ope_be", "gp_at", "ebit_sale", "at_gr1", "sale_gr1", "at_be","cash_at", "age", "z_score"]
data1 = data[cols1]
data1 = data1.dropna()

data_train = data1[(data1['date'] < 20120000)]
data_test = data1[(data1['date'] > 20120000)]

X_train_all = data_train[cols].to_numpy()
Y_train_all = data_train["ret_exc_lead1m"].to_numpy()

# split into training and validation
data_validation = data_train[(data_train['date'] > 20050000)]
data_train = data_train[(data_train['date'] < 20050000)]

X_train = data_train[cols].to_numpy()
Y_train = data_train["ret_exc_lead1m"].to_numpy()

X_test = data_test[cols].to_numpy()
Y_test = data_test["ret_exc_lead1m"].to_numpy()

X_validation = data_validation[cols].to_numpy()
Y_validation = data_validation["ret_exc_lead1m"].to_numpy()

P = [7,14,21]
D = [1,2,3]
M = [2,10]

p = 7
d = 1
m = 2

'''
for p in P:
    for d in D:
        for m in M:
            rf = RandomForestRegressor(n_estimators = 500, max_depth = d, min_samples_split = m, max_features = p, bootstrap = True, max_samples = 0.5)
# Train the model on training data
            rf.fit(X_train, Y_train)
            predictions = rf.predict(X_validation)
            mse_error = mean_squared_error(predictions, Y_validation)
# Print out the mean absolute error (mae)
            print('P=',p,' D=', d, ' M=', m ,'  Mean Squared Error:', round(mse_error, 6), 'degrees.')
'''

# we choose P =7, D = 1, M = 10

rf = RandomForestRegressor(n_estimators = 500, max_depth = 1, min_samples_split = 10, max_features = 7, bootstrap = True, max_samples = 0.5)
# Train the model on training data
rf.fit(X_train_all, Y_train_all)
predictions = rf.predict(X_test)
mse_error = mean_squared_error(predictions, Y_test)
print(mse_error)


from django.conf.urls import url
from . import views

urlpatterns = [
    url(r'^$', views.index, name='index'),
    url(r'^scrape_movies/$', views.scrape_movies, name='scrape_movies'),
    url(r'^view_movies/$', views.view_movies, name='view_movies'),
    url(r'^filter_movies/$', views.filter_movies, name='filter_movies'),
    url(r'^mark_read_movies/$', views.mark_read_movies, name='mark_read_movies'),
    url(r'^update_ratings/$', views.update_ratings, name='update_ratings'),
    url(r'^remove_duplicates/$', views.remove_duplicates, name='remove_duplicates'),
    url(r'^poll_state_scrape/$', views.poll_state_scrape, name='poll_state_scrape'),
    url(r'^poll_state_rating/$', views.poll_state_rating, name='poll_state_rating'),
    url(r'^poll_state_duplicates/$', views.poll_state_duplicates, name='poll_state_duplicates'),
    url(r'^search_movies/$', views.search_movies, name='search_movies'),
    url(r'^mark_read_bulk/$', views.mark_read_bulk, name='mark_read_bulk'),
    url(r'^delete_bulk/$', views.delete_bulk, name='delete_bulk'),
]


# Lifted heavily from Mustard Mine, and may have some unnecessary guff
import base64
import collections
import datetime
import functools
import json
import os
import sys
import threading
import time
import pytz
from pprint import pprint, pformat
# Hack: Get gevent to do its monkeypatching as early as possible.
# I have no idea what this is actually doing, but if you let the
# patching happen automatically, it happens too late, and we get
# RecursionErrors and such. There's a helpful warning on startup.
from gevent import monkey; monkey.patch_all(subprocess=True)
from flask import Flask, request, redirect, session, url_for, g, render_template, jsonify, Response, Markup
from authlib.integrations.requests_client import OAuth2Session
import requests

try:
	import config
except ImportError:
	# Construct a config object out of the environment
	import config_sample as config
	failed = []
	# Hack: Some systems like to give us a DATABASE_URL instead of a DATABASE_URI
	if "DATABASE_URL" in os.environ: os.environ["DATABASE_URI"] = os.environ["DATABASE_URL"]
	for var in dir(config):
		if var.startswith("__"): continue # Ignore dunders
		if var in os.environ: setattr(config, var, os.environ[var])
		else: failed.append(var)
	if failed:
		print("Required config variables %s not found - see config_sample.py" % ", ".join(failed), file=sys.stderr)
		sys.exit(1)
	sys.modules["config"] = config # Make the config vars available elsewhere

import database
app = Flask(__name__)
app.secret_key = config.SESSION_SECRET or base64.b64encode(os.urandom(12))

# Override Flask's forcing of Location headers to be absolute, since it
# gets stuff flat-out wrong. Also, the spec now says that relative
# headers are fine (and even when the spec said that the Location should
# to be absolute, everyone accepted relative URIs).
if os.environ.get("OVERRIDE_REDIRECT_HTTPS"):
	from werkzeug.middleware.proxy_fix import ProxyFix
	app.wsgi_app = ProxyFix(app.wsgi_app) # Grab info from Forwarded headers
	_redirect = redirect
	def redirect(*a, **kw):
		resp = _redirect(*a, **kw)
		resp.autocorrect_location_header = False
		return resp
	_url_for = url_for
	def url_for(*a, **kw): return _url_for(*a, **kw).replace("http://", "https://")

REQUIRED_SCOPES = "channel:read:subscriptions channel_subscriptions" # Ensure that these are sorted

class TwitchDataError(Exception):
	def __init__(self, error):
		self.__dict__.update(error)
		super().__init__(error["message"])

def query(endpoint, *, token, method="GET", params=None, data=None, auto_refresh=True):
	# If this is called outside of a Flask request context, be sure to provide
	# the auth token, and set auto_refresh to False.
	# TODO: Tidy up all this mess of auth patterns. It'll probably be easiest
	# to migrate everything to Helix first, and then probably everything will
	# use Bearer or App authentication.
	if token is None:
		auth = None
	elif token == "oauth":
		auth = "OAuth " + session["twitch_token"]
	elif token == "bearer":
		auth = "Bearer " + session["twitch_token"]
	elif token == "app":
		r = requests.post("https://id.twitch.tv/oauth2/token", data={
			"grant_type": "client_credentials",
			"client_id": config.CLIENT_ID, "client_secret": config.CLIENT_SECRET,
		})
		r.raise_for_status()
		data = r.json()
		auth = "Bearer " + data["access_token"]
		# TODO: Save the token so long as it's valid
		# expires = int(time.time()) + data["expires_in"] - 120
	else:
		auth = "OAuth " + token

	if not endpoint.startswith(("kraken/", "helix/")): raise ValueError("Need explicit selection of API (helix or kraken)")
	r = requests.request(method, "https://api.twitch.tv/" + endpoint,
		params=params, data=data, headers={
		"Accept": "application/vnd.twitchtv.v5+json",
		"Client-ID": config.CLIENT_ID,
		"Authorization": auth,
	})
	if auto_refresh and r.status_code == 401 and r.json()["message"] == "invalid oauth token":
		r = requests.post("https://id.twitch.tv/oauth2/token", data={
			"grant_type": "refresh_token",
			"refresh_token": session["twitch_refresh_token"],
			"client_id": config.CLIENT_ID, "client_secret": config.CLIENT_SECRET,
		})
		r.raise_for_status()
		resp = r.json()
		session["twitch_token"] = resp["access_token"]
		session["twitch_refresh_token"] = resp["refresh_token"]

		# Recurse for simplicity. Do NOT pass the original token, and be sure to
		# prevent infinite loops by disabling auto-refresh. Otherwise, pass-through.
		# (But DO pass the token-passing mode.)
		return query(endpoint, token="bearer" if token == "bearer" else "oauth",
			method=method, params=params, data=data, auto_refresh=False)
	if r.status_code == 403:
		# TODO: What if it *isn't* of this form??
		raise TwitchDataError(json.loads(r.json()["message"]))
	r.raise_for_status()
	if r.status_code == 204: return {}
	return r.json()

@app.route("/")
def mainpage():
	dest = os.environ.get("REDIRECT_TO_OTHER")
	if dest: return redirect(dest)
	# NOTE: If we've *reduced* the required scopes, this will still force a re-login.
	# However, it'll be an easy login, as Twitch will recognize the existing auth.
	if "twitch_token" not in session or session.get("twitch_auth_scopes") != REQUIRED_SCOPES:
		return render_template("login.html")
	user = session["twitch_user"]
	return render_template("index.html",
		username=user["display_name"], subscribers=database.list_subscribers(user["_id"]),
	)

@app.route("/logout")
def logout():
	session.pop("twitch_token", None)
	session.pop("twitch_user", None)
	return redirect(url_for("mainpage"))

@app.route("/login")
def login():
	twitch = OAuth2Session(config.CLIENT_ID, config.CLIENT_SECRET,
		scope=REQUIRED_SCOPES)
	uri, state = twitch.create_authorization_url("https://id.twitch.tv/oauth2/authorize",
		redirect_uri=os.environ.get("OVERRIDE_REDIRECT_URI") or url_for("authorized", _external=True))
	session["login_state"] = state
	return redirect(uri)

@app.route("/login/authorized")
def authorized():
	if "error" in request.args:
		# User cancelled the auth flow - discard auth (most likely there won't be any)
		session.pop("twitch_token", None)
		return redirect(url_for("mainpage"))
	twitch = OAuth2Session(config.CLIENT_ID, config.CLIENT_SECRET,
		state=session["login_state"])
	resp = twitch.fetch_access_token("https://id.twitch.tv/oauth2/token",
		code=request.args["code"],
		# For some bizarre reason, we need to pass this information along.
		client_id=config.CLIENT_ID, client_secret=config.CLIENT_SECRET,
		redirect_uri=os.environ.get("OVERRIDE_REDIRECT_URI") or url_for("authorized", _external=True))
	if "access_token" not in resp:
		# Something went wrong with the retrieval. No idea what or why,
		# so I'm doing a cop-out and just dumping to console.
		print("Unable to log in", file=sys.stderr)
		pprint(resp, stream=sys.stderr)
		print("Returning generic failure.", file=sys.stderr)
		raise Exception
	session["twitch_token"] = resp["access_token"]
	session["twitch_refresh_token"] = resp["refresh_token"]
	session["twitch_auth_scopes"] = " ".join(sorted(resp["scope"]))
	# kraken_user = query("kraken/user", token="oauth")
	# The Kraken response includes fields not in Helix, including created_at,
	# and email (though Helix gives us the latter if we add an OAuth scope).
	user = query("helix/users", token="bearer")["data"][0]
	user["_id"] = user["id"] # For now, everything looks for _id. Existing logins don't have user["id"].
	database.login_user(user["_id"], session["twitch_token"])
	session["twitch_user"] = user
	return redirect(url_for("mainpage"))

@app.route("/upload", methods=["POST"])
def upload_files():
	twitchid = session["twitch_user"]["_id"]
	for f in request.files.getlist("csv"):
		print("Loading", f.filename)
		database.bulk_load_subs(twitchid, f.read().decode("UTF-8"))
	return redirect(url_for("mainpage"))

# TODO: JSON API endpoints for uploading a CSV, and forcing a recheck

# Hack. TODO: Have a UI to do this
@app.route("/ping-api")
def ping_api():
	id = session["twitch_user"]["_id"]
	'''
	params = {"limit": 100, "offset": 0}
	subs = []
	while True:
		data = query("kraken/channels/%s/subscriptions" % id, token="oauth", params=params)
		if not data["subscriptions"]: break
		subs.extend(data["subscriptions"])
		params["offset"] += params["limit"]
	database.update_subs_from_api(id, subs)
	'''
	data = query("helix/subscriptions", token="bearer", params={"broadcaster_id": id})
	# TODO: Paginate the results
	return "<pre>" + pformat(data) + "</pre>"

if __name__ == "__main__":
	import logging
	logging.basicConfig(level=logging.INFO)
	# Load us up using gunicorn, configured via the Procfile
	with open("Procfile") as f: cmd = f.read().strip().replace("web: ", "")
	if "PORT" not in os.environ: os.environ["PORT"] = "5000" # hack - pick a different default port
	sys.argv = cmd.split(" ")[1:] # TODO: Split more smartly
	from gunicorn.app.wsgiapp import run; run()
else:
	# Worker startup. This is the place to put any actual initialization work
	# as it won't be done on master startup.
	pass


from typing import Optional

from pydantic import Field, HttpUrl, validator

from app.model.response import Success
from app.model.base import DataModel, InCreateModel, InUpdateModel
from app.database.table.location_cabinet import CabinetStatus
from app.util.type.guid import GUID
from app.util.regex_pattern import NAME_PATTERN
from app.util.string_length import SHORT_LENGTH, LONG_LENGTH, URL_LENGTH

class _BaseCabinet(DataModel):
    located_room: GUID = Field(
        ...,
        title="存储柜所在房间的 ID",
    )
    cabinet_name: str = Field(
        ...,
        max_length=SHORT_LENGTH,
        regex=NAME_PATTERN,
        title="存储柜名称",
        description="用来区分存储柜，只能使用中文、大小写字母、数字、下划线、中划线，默认为 Unnamed Cabinet",
        example="Cabinet_1",
    )
    cabinet_comment: Optional[str] = Field(
        None,
        max_length=LONG_LENGTH,
        title="存储柜备注",
        example="Some comment of the cabinet",
    )
    cabinet_image_url: Optional[HttpUrl] = Field(
        None,
        max_length=URL_LENGTH,
        title="存储柜图片的 URL",
        example="http://www.example.com/image.png",
    )
    max_number: int = Field(
        ...,
        gt=0,
        title="存储柜最大容量",
        description="存储柜最多能存放的物品数量，如果不希望被限制可以尝试使用一个非常大的数值",
        example=9999,
    )
    current_number: int = Field(
        ...,
        gt=0,
        title="存储柜当前容量",
        description="当前容量不能超过最大值，且当达到最大时会更新存储柜状态为满载",
        example=100,
    )
    status: CabinetStatus = Field(
        ...,
        title="存储柜状态",
        description="""
        可选的存储柜状态为：

            - DISABLED  (0): 禁用
            - ENABLED   (1): 启用
            - FULL_LOAD (2): 满载

        新建立的存储柜默认为禁用（ DISABLED ），可以在设置中修改。""",
    )

    @validator("current_number")
    def check_current_number(cls, value: int, values: dict) -> int:
        """检查存储柜当前容量

        Args:
            value (int): 当前容量
            values (dict): 模型全部字段

        Raises:
            ValueError: 当当前容量大于最大容量时抛出异常

        Returns:
            int: 当前容量
        """
        max_number: int | None = values.get("max_number")

        assert max_number is not None, "max_number is required"

        if value > max_number:
            raise ValueError(
                f"current_number ({value}) is greater than max_number ({max_number})"
            )
        return value

    @validator("status")
    def check_status(cls, value: CabinetStatus, values: dict) -> CabinetStatus:
        """检查存储柜状态

        Args:
            value (CabinetStatus): 当前的状态
            values (dict): 模型全部字段

        Raises:
            ValueError: 当输入的状态不支持时抛出异常

        Returns:
            CabinetStatus: 存储柜状态
        """
        max_number: int | None = values.get("max_number")
        current_number: int | None = values.get("current_number")

        assert max_number is not None, "max_number is required"
        assert current_number is not None, "current_number is required"

        if max_number == current_number:
            return CabinetStatus.FULL_LOAD
        return value

class Cabinet(_BaseCabinet):
    pass

class CabinetInCreate(InCreateModel, _BaseCabinet):
    cabinet_name: Optional[str] = Field(
        "Unnamed Cabinet",
        max_length=SHORT_LENGTH,
        regex=NAME_PATTERN,
        title="存储柜名称",
        description="用来区分存储柜，只能使用中文、大小写字母、数字、下划线、中划线，默认为 Unnamed Cabinet",
        example="Cabinet_1",
    )
    current_number: Optional[int] = Field(
        0,
        title="存储柜当前容量",
        description="当前容量不能超过最大值，且当达到最大时会更新存储柜状态为满载",
        example=100,
    )
    status: Optional[CabinetStatus] = Field(
        CabinetStatus.DISABLED,
        title="存储柜状态",
        description="""
        可选的存储柜状态为：

            - DISABLED  (0): 禁用
            - ENABLED   (1): 启用
            - FULL_LOAD (2): 满载

        新建立的存储柜默认为禁用（ DISABLED ），可以在设置中修改。""",
    )

class CabinetInUpdate(InUpdateModel, _BaseCabinet):
    located_room: Optional[GUID] = Field(
        None,
        title="存储柜所在房间的 ID",
    )
    cabinet_name: Optional[str] = Field(
        None,
        max_length=SHORT_LENGTH,
        regex=NAME_PATTERN,
        title="存储柜名称",
        description="用来区分存储柜，只能使用中文、大小写字母、数字、下划线、中划线，默认为 Unnamed Cabinet",
        example="Cabinet_1",
    )
    cabinet_comment: Optional[str] = Field(
        None,
        max_length=LONG_LENGTH,
        title="存储柜备注",
        example="Some comment of the cabinet",
    )
    cabinet_image_url: Optional[HttpUrl] = Field(
        None,
        max_length=URL_LENGTH,
        title="存储柜图片的 URL",
        example="http://www.example.com/image.png",
    )
    max_number: Optional[int] = Field(
        None,
        gt=0,
        title="存储柜最大容量",
        description="存储柜最多能存放的物品数量，如果不希望被限制可以尝试使用一个非常大的数值",
        example=9999,
    )
    current_number: Optional[int] = Field(
        None,
        gt=0,
        title="存储柜当前容量",
        description="当前容量不能超过最大值，且当达到最大时会更新存储柜状态为满载",
        example=100,
    )
    status: Optional[CabinetStatus] = Field(
        None,
        title="存储柜状态",
        description="""
        可选的存储柜状态为：

            - DISABLED  (0): 禁用
            - ENABLED   (1): 启用
            - FULL_LOAD (2): 满载

        新建立的存储柜默认为禁用（ DISABLED ）。""",
    )

class CabinetInResponse(Success):
    data: list[Cabinet]


#
# matplot graph class definition
#
# backend independent implementation
#
# Kazutomo Yoshii <ky@anl.gov>
#

import os, sys

import matplotlib.pyplot as plt
import matplotlib.animation as manimation
#import matplotlib.collections as collections
import matplotlib.cm as cm
from matplotlib.offsetbox import OffsetImage, AnnotationBbox
from matplotlib.cbook import get_sample_data
from matplotlib._png import read_png

from listrotate import *

class plot_info:
    def ypos(self,i):
        return 1.0 - 0.05*i

    def __init__(self, ax, params):
        self.ax = ax

        dir=os.path.abspath(os.path.dirname(sys.argv[0]))

        ax.axis([0,1,0,1])
        ax.axes.get_xaxis().set_visible(False)
        ax.axes.get_yaxis().set_visible(False)
        ax.set_frame_on=True

        cfg = params['cfg']
        info = params['info']

        xoff = 0.05

        l=1
#        for p in range(info['npkgs']):
#           ax.text( 0.1, self.ypos(l), 'CPU PKG%d' % p, color=params['pkgcolors'][p] )
#            l += 1
#        l += 2

        ax.text(xoff, self.ypos(l), '[INFO]' )
        l += 2
        ax.text(xoff, self.ypos(l), 'Description : %s' % cfg['desc'] )
        l += 1
        ax.text(xoff, self.ypos(l), 'Node : %s' % info['nodeinfo'] )
        l += 1
        ax.text(xoff, self.ypos(l), 'Linux kernel : %s' % info['kernelversion'] )
        l += 1
        plt.text( xoff, self.ypos(l), 'Freq. driver : %s' % info['freqdriver'] )
        l += 1
        plt.text( xoff, self.ypos(l), 'Memory [GB] : %d' % (int(info['memoryKB'])/1024/1024) )
        l += 1
        plt.text( xoff, self.ypos(l), 'CPU model : %s' % info['cpumodel'] )
        l += 1
        plt.text( xoff, self.ypos(l), 'No of procs : %s' % info['ncpus'] )
        l += 1
        plt.text( xoff, self.ypos(l), 'No of pkgs : %s' % info['npkgs'] )
        l += 1
        plt.text( xoff, self.ypos(l), 'No of NUMA nodes: %d' % info['nnodes'] )

        a = info['pkg0phyid']
        ht = 'enabled'
        if len(a) == len(set(a)):
            ht = 'disabled'
        l += 1
        plt.text( xoff, self.ypos(l), 'Hyperthread : %s' % ht)
#        l += 1
#        plt.text( xoff, ypos(l), 'Powercap pkg0 : %d Watt' % s_powercap['p0'] )
#        l += 1
#        plt.text( xoff, ypos(l), 'Powercap pkg1 : %d Watt' % s_powercap['p1'] )
#        l += 1

        fn = get_sample_data("%s/coolr-logo-poweredby-48.png" % dir, asfileobj=False)
        arr = read_png(fn)
        imagebox = OffsetImage(arr, zoom=0.6)
        ab = AnnotationBbox(imagebox, (0, 0),
                            xybox=(.8, .1),
                            xycoords='data',
                            boxcoords="axes fraction",
                            pad=0.5)
        ax.add_artist(ab)

    def update(self, n):
        # self.text1.set_text('%d' % n)
        s = 'dummy'

class plot_totpwr:
    def __init__(self, ax, params, totpwrs):
        self.ax = ax
        self.update(params, totpwrs)

    def update(self, params, totpwrs):

        cfg = params['cfg']
        cur_t = params['cur']
        gxsec = params['gxsec']

        self.ax.cla() # this is a brute-force way to update

        self.ax.axis([cur_t-gxsec, cur_t, cfg['pwrmin'], cfg['acpwrmax']]) # [xmin,xmax,ymin,ymax]

        x = totpwrs[0].getlistx()
        y = totpwrs[0].getlisty()
        self.ax.plot(x,y, scaley=False, color='black', label='RAPL total' )
        if len(totpwrs) > 1:
            x = totpwrs[1].getlistx()
            y = totpwrs[1].getlisty()
            self.ax.plot(x,y, '--', scaley=False,  color='black', label='AC' )

        self.ax.legend(loc='lower left', prop={'size':9})
        self.ax.set_xlabel('Time [S]')
        self.ax.set_ylabel('Power [W]')

# the following plots can be generalized
class plot_xsbench:
    def __init__(self, ax, params, lps):
        self.ax = ax

        # too lazy to figure out axhspan's object. fix this later
        self.update(params, lps)

    def update(self, params, lps):

        cfg = params['cfg']
        cur_t = params['cur']
        gxsec = params['gxsec']

        self.ax.cla() # this is a brute-force way to update

        self.ax.set_xlim([cur_t-gxsec, cur_t])
        self.ax.set_ylim(bottom=0)

        x = lps.getlistx()
        y = lps.getlisty()
        #self.ax.plot(x,y, scaley=False, color='black', label='' )
        self.ax.bar(x,y,  color='black', label='' )

        self.ax.legend(loc='lower left', prop={'size':9})
        self.ax.set_xlabel('Time [S]')
        self.ax.set_ylabel('TTS [S]')

class plot_appperf:
    def __init__(self, ax, params, lps):
        self.ax = ax

        # too lazy to figure out axhspan's object. fix this later
        self.update(params, lps)

    def update(self, params, lps):

        cfg = params['cfg']
        cur_t = params['cur']
        gxsec = params['gxsec']

        self.ax.cla() # this is a brute-force way to update

        self.ax.set_xlim([cur_t-gxsec, cur_t])
        self.ax.autoscale_view(scaley=True)
        self.ax.set_ylim(bottom=0)

        x = lps.getlistx()
        y = lps.getlisty()
        self.ax.plot(x,y, label='')

#        self.ax.legend(loc='lower left', prop={'size':9})
        self.ax.set_xlabel('Time [S]')
        self.ax.set_ylabel('App performance')

class plot_runtime: # mean, std
    def __init__(self, ax, params, pdata):
        self.ax = ax
        self.update(params, pdata)

    def update(self, params, pdata, ptype = 'temp'):
        cfg = params['cfg']
        cur_t = params['cur']
        gxsec = params['gxsec']

        self.ax.cla()
        self.ax.set_xlim([cur_t-gxsec, cur_t])
        self.ax.autoscale_view(scaley=True)

        x = pdata.getlistx()
        y = pdata.getlisty()
        e = pdata.getlisto()
        self.ax.plot(x,y, scaley=True,  label='')
        self.ax.errorbar(x,y,yerr=e, lw=.2,  label = '')

        # we need to update labels everytime because of cla()
        self.ax.set_xlabel('Time [S]')
        self.ax.set_ylabel('Runtime')
#        self.ax.legend(loc='lower left', prop={'size':9})
# ----------------------

class plot_rapl:
    def __init__(self, ax, params, ppkg, pmem, titlestr=''):
        self.ax = ax
        self.titlestr = titlestr
        # too lazy to figure out axhspan's object. fix this later
        self.update(params, ppkg, pmem)

    def update(self, params, ppkg, pmem):

        cfg = params['cfg']
        cur_t = params['cur']
        gxsec = params['gxsec']

        self.ax.cla() # this is a brute-force way to update

        self.ax.set_xlim([cur_t-gxsec, cur_t])
        self.ax.autoscale_view(scaley=True)

        #self.ax.axis([cur_t-gxsec, cur_t, cfg['pwrmin'], cfg['pwrmax']]) # [xmin,xmax,ymin,ymax]

        pkgid = 0
        for t in ppkg:
            x = t.getlistx()
            ycap = t.getlisto()
            #self.ax.plot(x,ycap, scaley=False, color='red', label='PKG%dlimit'%pkgid )
            self.ax.plot(x,ycap, color='red', label='PKG%dlimit'%pkgid )
            pkgid += 1

        pkgid = 0
        for t in ppkg:
            x = t.getlistx()
            y = t.getlisty()
            #self.ax.plot(x,y,scaley=False,color=params['pkgcolors'][pkgid], label='PKG%d'%pkgid)
            self.ax.plot(x,y,color=params['pkgcolors'][pkgid], label='PKG%d'%pkgid)
            pkgid += 1

        pkgid = 0
        for t in pmem:
            x = t.getlistx()
            y = t.getlisty()
            #self.ax.plot(x,y,scaley=False,color=params['pkgcolors'][pkgid], linestyle='--', label='PKG%ddram'%pkgid)
            self.ax.plot(x,y,color=params['pkgcolors'][pkgid], linestyle='--', label='PKG%ddram'%pkgid)
            pkgid += 1

        self.ax.legend(loc='lower left', prop={'size':9})
        self.ax.set_xlabel('Time [S]')
        self.ax.set_ylabel('Power [W]')
        if len(self.titlestr):
            self.ax.set_title("%s" % self.titlestr)

class plot_line_err: # used for temp and freq (mean+std)
    def __init__(self, ax, params, pdata, ptype = 'temp' ):
        self.ax = ax

        # unfortunately, I couldn't figure out how to update errorbar correctly
        self.update(params, pdata, ptype)

    def update(self, params, pdata, ptype = 'temp'):
        cfg = params['cfg']
        cur_t = params['cur']
        gxsec = params['gxsec']

        self.ax.cla() # this is a brute-force way to update. I don't know how to update errorbar correctly.
        if ptype == 'temp':
            self.ax.axis([cur_t-gxsec, cur_t, cfg['tempmin'], cfg['tempmax']]) # [xmin,xmax,ymin,ymax]
        elif ptype == 'freq':
            self.ax.axis([cur_t-gxsec, cur_t, cfg['freqmin'], cfg['freqmax']]) # [xmin,xmax,ymin,ymax]
            plt.axhspan(cfg["freqnorm"], cfg["freqmax"], facecolor='#eeeeee', alpha=0.5)
        else:
            self.ax.axis([cur_t-gxsec, cur_t, 0, 100]) # [xmin,xmax,ymin,ymax]

        pkgid = 0
        for t in pdata:
            x = t.getlistx()
            y = t.getlisty()
            e = t.getlisto()
            self.ax.plot(x,y,scaley=False,color=params['pkgcolors'][pkgid], label='PKG%d'%pkgid)
            self.ax.errorbar(x,y,yerr=e, lw=.2, color=params['pkgcolors'][pkgid], label = '')
            pkgid += 1

        # we need to update labels everytime because of cla()
        self.ax.set_xlabel('Time [S]')
        if ptype == 'temp':
            self.ax.set_ylabel('Core temperature [C]')
        elif ptype == 'freq':
            self.ax.set_ylabel('Frequency [GHz]')
        else:
            self.ax.set_ylabel('Unknown')
        self.ax.legend(loc='lower left', prop={'size':9})

# below are kind of examples
#

class plotline:
    def __init__(self, ax, x, y):
        self.ax = ax
        self.line, = ax.plot(x,y)

        self.ax.axhspan( 0.7, 1.0, facecolor='#eeeeee', alpha=1.0)

    def update(self, x, y):
        self.line.set_data(x, y)

class plotcolormap:
    def __init__(self, ax, X):
        self.ax = ax
        self.im = self.ax.imshow(X, cmap=cm.jet, interpolation='nearest')
        self.im.set_cmap('spectral')
        self.im.set_clim(0, 1.5)

        f = plt.gcf()
        f.colorbar(self.im)

    def update(self,X):
        self.im.set_array(X)

class plotbar:
    def __init__(self, ax, x, y):
        self.ax = ax
        self.rects = ax.bar(x, y)

    def update(self, y):
        for r, h in zip(self.rects, y):
            r.set_height(h)

class ploterrorbar:
    def __init__(self, ax, x, y, e):
        self.ax = ax
        l, (b, t), v = ax.errorbar(x, y, e)
        self.line = l
        self.bottom = b
        self.top = t
        self.vert = v

    def update(self, x, y, e):
        # XXX: this is a bit brute-force
        # I couldn't figure out how to update vert
        self.ax.cla()
        self.ax.errorbar(x, y, e)

class plottext:
    def ypos(self,i):
        return 1.0 - 0.05*i

    def __init__(self, ax, n):
        self.ax = ax

        dir=os.path.abspath(os.path.dirname(sys.argv[0]))

        ax.axis([0,1,0,1])
        ax.axes.get_xaxis().set_visible(False)
        ax.axes.get_yaxis().set_visible(False)
        ax.set_frame_on=True

        ax.plot( [ 0.1, 0.2], [0.96, 0.96], color='blue',  linewidth=2 )
        ax.plot( [ 0.1, 0.2], [0.91, 0.91], color='green', linewidth=2 )
        ax.plot( [ 0.1, 0.2], [0.86, 0.86], color='red',   linewidth=1 )

        self.text1 = ax.text( 0.3, self.ypos(2), '%d' % n )

        fn = get_sample_data("%s/coolr-logo-poweredby-48.png" % dir, asfileobj=False)
        arr = read_png(fn)
        imagebox = OffsetImage(arr, zoom=0.4)
        ab = AnnotationBbox(imagebox, (0, 0),
                            xybox=(.75, .12),
                            xycoords='data',
                            boxcoords="axes fraction",
                            pad=0.5)
        ax.add_artist(ab)

    def update(self, n):
        self.text1.set_text('%d' % n)


# -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Nicole Thomas <nicole@saltstack.com>`
'''

# Import Python Libs
import os
import random
import string

# Import Salt Testing Libs
from salttesting import skipIf
from salttesting.helpers import ensure_in_syspath, expensiveTest

ensure_in_syspath('../../../')

# Import Salt Libs
import integration
from salt.config import cloud_providers_config

# Import Third-Party Libs
try:
    import libcloud  # pylint: disable=W0611
    HAS_LIBCLOUD = True
except ImportError:
    HAS_LIBCLOUD = False

def __random_name(size=6):
    '''
    Generates a random cloud instance name
    '''
    return 'CLOUD-TEST-' + ''.join(
        random.choice(string.ascii_uppercase + string.digits)
        for x in range(size)
    )

# Create the cloud instance name to be used throughout the tests
INSTANCE_NAME = __random_name()

@skipIf(HAS_LIBCLOUD is False, 'salt-cloud requires >= libcloud 0.13.2')
class RackspaceTest(integration.ShellCase):
    '''
    Integration tests for the Rackspace cloud provider using the Openstack driver
    '''

    @expensiveTest
    def setUp(self):
        '''
        Sets up the test requirements
        '''
        super(RackspaceTest, self).setUp()

        # check if appropriate cloud provider and profile files are present
        profile_str = 'rackspace-config:'
        provider = 'rackspace'
        providers = self.run_cloud('--list-providers')
        if profile_str not in providers:
            self.skipTest(
                'Configuration file for {0} was not found. Check {0}.conf files '
                'in tests/integration/files/conf/cloud.*.d/ to run these tests.'
                .format(provider)
            )

        # check if api key, user, and tenant are present
        path = os.path.join(integration.FILES,
                            'conf',
                            'cloud.providers.d',
                            provider + '.conf')
        config = cloud_providers_config(path)
        user = config['rackspace-config']['openstack']['user']
        tenant = config['rackspace-config']['openstack']['tenant']
        api = config['rackspace-config']['openstack']['apikey']
        if api == '' or tenant == '' or user == '':
            self.skipTest(
                'A user, tenant, and an api key must be provided to run these '
                'tests. Check tests/integration/files/conf/cloud.providers.d/{0}.conf'
                .format(provider)
            )

    def test_instance(self):
        '''
        Test creating an instance on rackspace with the openstack driver
        '''

        # create the instance
        instance = self.run_cloud('-p rackspace-test {0}'.format(INSTANCE_NAME))
        ret = '        {0}'.format(INSTANCE_NAME)

        # check if instance with salt installed returned successfully
        try:
            self.assertIn(ret, instance)
        except AssertionError:
            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME))
            raise

        # delete the instance
        delete = self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME))
        ret = '            True'
        try:
            self.assertIn(ret, delete)
        except AssertionError:
            raise

    def tearDown(self):
        '''
        Clean up after tests
        '''
        query = self.run_cloud('--query')
        ret = '        {0}:'.format(INSTANCE_NAME)

        # if test instance is still present, delete it
        if ret in query:
            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME))

if __name__ == '__main__':
    from integration import run_tests
    run_tests(RackspaceTest)


import re

from django import forms
from django.conf import settings
from django.contrib.auth.models import User
from django.utils.translation import ugettext as _

from base.models import Lexicon, EXCLUDED_LEXICA

class ProfileEditForm(forms.Form):
    lexiconChoices = Lexicon.objects.exclude(lexiconName__in=EXCLUDED_LEXICA)
    defaultLexicon = forms.ModelChoiceField(
        queryset=lexiconChoices,
        label='Default Lexicon',
        widget=forms.Select(attrs={'class': 'form-control'}),
        empty_label=None)
    profileText = forms.CharField(
        widget=forms.Textarea(attrs={'class': 'form-control'}),
        label='Your profile',
        required=False)
    disableChat = forms.BooleanField(label='Disable Chat', required=False)
    default_language = forms.ChoiceField(
        choices=settings.LANGUAGES,
        required=False,
        widget=forms.Select(attrs={'class': 'form-control'}),
    )

class UsernameEditForm(forms.Form):
    username = forms.CharField(label='Your desired username', required=True)

    def clean(self):
        u = self.cleaned_data.get('username')
        if not u:
            raise forms.ValidationError(_('You must enter a username'))
        if not re.match(r'\w+$', u):
            raise forms.ValidationError(
                _('Your username must consist of alphanumeric characters.'))
        # Case-insensitive username match
        users = User.objects.filter(username__iexact=u)
        if users.count() > 0:
            raise forms.ValidationError(
                _('This username already exists in our system!'))


# https://leetcode.com/problems/merge-two-sorted-lists/
# Definition for singly-linked list.
from typing import Optional

class ListNode:
    def __init__(self, val=0, next=None):
        self.val = val
        self.next = next

class Solution:
    def mergeTwoLists(self, list1: Optional[ListNode], list2: Optional[ListNode]) -> Optional[ListNode]:
        head = tail = ListNode(0)

        while list1 and list2:
            if list1.val < list2.val:
                tail.next = list1
                list1 = list1.next
            else:
                tail.next = list2
                list2 = list2.next
            tail = tail.next

        tail.next = list1 or list2
        return head.next

if __name__ == '__main__':
    l1 = ListNode(1)
    l1.next = ListNode(2)
    l1.next = ListNode(4)
    l2 = ListNode(1)
    l2.next = ListNode(3)
    l2.next = ListNode(4)

    solution = Solution()
    print(solution.mergeTwoLists(l1, l2))