import os
import sys
sys.path.append(os.getcwd())
from functools import partial
import os
import pickle
import sys
import torch
from copy import deepcopy
import numpy as np
import matplotlib.pyplot as plt
import csv
from collections import defaultdict
import math
import glob
import re

from safe_control_gym.utils.configuration import ConfigFactory
from safe_control_gym.utils.plotting import plot_from_logs, window_func, load_from_log_file, COLORS, LINE_STYLES
from safe_control_gym.utils.registration import make
from safe_control_gym.utils.utils import mkdirs, set_dir_from_config, set_device_from_config, set_seed_from_config, save_video

from eval_utils import *

##############################################################################################
#############################   Constants   ##################################################
##############################################################################################

# quadrotor controller time interval
dt = 1.0 / 50.0

##############################################################################################
#############################   Testing Functions   ##########################################
##############################################################################################

def test_policy(config):
    """Run the (trained) policy/controller for evaluation.

    Saves test trajectories to csv files.
    Supports evaluation with fixed initial conditions.

    """
    assert config.restore is not None, "must specify --restore for policy evaluation."
    assert config.task_config.done_on_out_of_bound is False, "post-evaluation should disable early termination."
    # Evaluation setup.
    set_device_from_config(config)
    if config.set_test_seed:
        # seed the evaluation (both controller and env) if given
        set_seed_from_config(config)
        env_seed = config.seed
    else:
        env_seed = None
    # Define function to create task/env.
    env_func = partial(make, config.task, seed=env_seed, output_dir=config.output_dir, **config.task_config)
    # Create the controller/control_agent.
    control_agent = make(config.algo,
                         env_func,
                         training=False,
                         checkpoint_path=os.path.join(config.output_dir, "model_latest.pt"),
                         output_dir=config.output_dir,
                         device=config.device,
                         seed=config.seed,
                         **config.algo_config)
    control_agent.reset()
    if config.restore:
        control_agent.load(os.path.join(config.restore, config.restore_model_file))

    # Test controller.
    if config.fix_env_init:
        n_episodes = 1
        # quad has default 0 for initial states
        init_state = {}
        env = make_fixed_init_env(config, init_state=init_state,
                                  env_seed=env_seed, n_episodes=n_episodes)
    else:
        n_episodes = config.algo_config.eval_batch_size
        env = control_agent.env
    results = run_with_env(config, control_agent, env, n_episodes=n_episodes,
                           render=config.render, verbose=config.verbose)

    # Save evalution results.
    if hasattr(config, "eval_output_dir") and config.eval_output_dir is not None:
        eval_output_dir = config.eval_output_dir
    else:
        eval_output_dir = os.path.join(config.output_dir, "eval")
    os.makedirs(eval_output_dir, exist_ok=True)
    # test trajs and statistics
    eval_path = os.path.join(eval_output_dir, config.eval_output_path)
    os.makedirs(os.path.dirname(eval_path), exist_ok=True)
    with open(eval_path, "wb") as f:
        pickle.dump(results, f)
    # time and phase trajs for better visualization & plotting
    save_traj_to_csv(config, env, results, os.path.join(eval_output_dir, "eval_trajs"))
    plot_time_traj(env, results,
                   output_path=os.path.join(eval_output_dir, "time_trajectory.png"),
                   max_episode_plot=config.max_episode_plot)
    plot_phase_traj(env, results,
                   output_path=os.path.join(eval_output_dir, "phase_trajectory.png"),
                   max_episode_plot=config.max_episode_plot)
    control_agent.close()
    print("Evaluation done.")

def test_from_checkpoints(config):
    """Use saved checkpoints to test policy throughout training.

    This function is used when evaluation uses different settings to that during training.
    Hence, we use saved checkpoints to run test episodes with the modified settings,
    e.g. here we calculate the normalized metrics, including the RMSE as a common metric
    to compare fairly with other control baselines.

    """
    assert config.restore is not None, "must specify --restore for policy evaluation."
    assert config.task_config.done_on_out_of_bound is False, "post-evaluation should disable early termination."
    # Get all checkpoints
    checkpoint_dir_full = os.path.join(config.restore, config.checkpoint_dir)
    assert os.path.exists(checkpoint_dir_full), "checkpoint_dir does not exist in {}".format(config.restore)
    checkpoint_re = re.compile("model_(.*)\.pt")
    checkpoints = [
        [float(checkpoint_re.search(pt).group(1)), pt]
        for pt in os.listdir(checkpoint_dir_full)
    ]
    checkpoints = sorted(checkpoints, key=lambda x: x[0])

    # Evaluation setup.
    set_device_from_config(config)
    if config.set_test_seed and config.set_test_seed_as_training_eval:
        raise ValueError("Can only enable either `set_test_seed` or `set_test_seed_as_training_eval`.")
    elif config.set_test_seed:
        set_seed_from_config(config)
        env_seed = config.seed
    elif config.set_test_seed_as_training_eval:
        # algo seeding uses seed for original training
        set_seed_from_config(config)
        # env seeding uses seed for eval_env back in during training
        env_seed = config.seed * 111
    else:
        env_seed = None

    # Define function to create task/env.
    env_func = partial(make, config.task, seed=env_seed, output_dir=config.output_dir, **config.task_config)
    # create env and send to .run() explicitly
    # since if not, the default self.env has infinit buffer but not of size eval_batch_size
    env = env_func()
    # Create the controller/control_agent.
    control_agent = make(config.algo,
                         env_func,
                         training=False,
                         checkpoint_path=os.path.join(config.output_dir, "model_latest.pt"),
                         output_dir=config.output_dir,
                         device=config.device,
                         seed=config.seed,
                         **config.algo_config)
    control_agent.reset()

    # Test on each checkpoint
    test_results = defaultdict(list)
    for i, (step, checkpoint) in enumerate(checkpoints):
        print("Progress {}, step {} evaluating...".format(i / len(checkpoints), step))
        checkpoint_path_full = os.path.join(checkpoint_dir_full, checkpoint)
        control_agent.load(checkpoint_path_full)
        results = control_agent.run(env=env, n_episodes=config.algo_config.eval_batch_size)

        test_results["step"].append(step)
        ep_lengths = results["ep_lengths"]
        for k, v in results.items():
            if not isinstance(v, np.ndarray):
                pass
            # raw mean over n_episodes
            test_results[k].append(v.mean())
            # normalized mean
            if k is not "ep_lengths":
                assert len(v) == len(ep_lengths)
                # don't use `v / ep_lengths`, in case v is more than 1 dim
                normalized_v = np.asarray([v[i] / ep_lengths[i] for i in range(len(v))])
                if "mse" in k:
                    # convert mse cost to rmse (as we desired for common metric across baselines)
                    stat = np.mean(np.sqrt(normalized_v))
                    name = k.replace("mse", "rmse")
                else:
                    # take simple mean for other metrics such as constrasint violations
                    stat = np.mean(normalized_v)
                    name = k
                test_results["normalized_" + name].append(stat)

    # Save evalution results.
    checkpoint_eval_dir_full = os.path.join(config.restore, "logs", config.checkpoint_eval_dir)
    os.makedirs(checkpoint_eval_dir_full, exist_ok=True)
    steps = test_results.pop("step")

    for k, v in test_results.items():
        scalar_name = "{}/{}".format(config.checkpoint_eval_dir, k)
        header = ["step", scalar_name]
        stat_mtx = np.array([steps, v]).transpose()
        rows = stat_mtx.tolist()

        csv_path = os.path.join(checkpoint_eval_dir_full, "{}.log".format(k))
        with open(csv_path, "w") as f:
            writer = csv.writer(f)
            writer.writerow(header)
            writer.writerows(rows)
    control_agent.close()
    print("Checkpoint evaluation done.")

##############################################################################################
#############################   Performance   ##################################################
##############################################################################################

def plot_performance(config):
    """Gets the plot and csv for performance (in RMSE)."""
    legend_map = {
        "ppo": "ppo",
        "sac": "sac",
    }
    algo_name_map = {
        "ppo": "PPO",
        "sac": "SAC",
    }
    scalar_name_map = {
        "checkpoint_eval/normalized_rmse": "Cost",
    }

    # Collect results.
    spec = {}
    for d, legend in legend_map.items():
        seed_dirs = os.listdir(os.path.join(config.plot_dir, d))
        spec[legend] = [os.path.join(config.plot_dir, d, sd) for sd in seed_dirs]

    # Collective plot (mean only).
    scalar_stats = plot_from_exps(
        spec,
        out_path=os.path.join(config.plot_dir, "performance.jpg"),
        scalar_names=["checkpoint_eval/normalized_rmse"],
        title="Learning Curves",
        sub_titles=["Average RMSE"],
        xlabel="Training Time (s)",
        ylabels=["Cost"],
        window=None,
        x_num_max=None,
        x_rescale_factor=dt,
        num_std=0,
        use_median_quantile=True,
        cols_per_row=3
    )

    # Save stats to CSV.
    save_stats_to_csv(
        scalar_stats,
        algo_name_map=algo_name_map,
        scalar_name_map=scalar_name_map,
        csv_path=os.path.join(config.plot_dir, "performance.csv")
    )
    print("Performance plotting done.")

##############################################################################################
#############################   Constraint   ##################################################
##############################################################################################

def plot_constraint(config):
    """"""
    legend_map = {
        "ppo": "ppo",
        "safe_explorer_ppo/slack0.02": "seppo_0.02",
        # "safe_explorer_ppo/slack0.02_lr0.0001": "seppo_0.02_lr0.0001",
    }
    name_map = {
        "ppo": "PPO",
        "safe_explorer_ppo/slack0.02": "PPO safety_layer slack0.02",
        # "safe_explorer_ppo/slack0.02_lr0.0001": "PPO safety_layer slack0.02 lr 0.0001",
    }
    scalar_name_map = {
        "checkpoint_eval/normalized_constraint_violation": "Constraint Violations",
    }

    # Collect results.
    spec = {}
    for d, legend in legend_map.items():
        seed_dirs = os.listdir(os.path.join(config.plot_dir, d))
        spec[legend] = [os.path.join(config.plot_dir, d, sd) for sd in seed_dirs]

    # Collective plot (mean only).
    scalar_stats = plot_from_exps(
        spec,
        out_path=os.path.join(config.plot_dir, "constraint_performance.jpg"),
        scalar_names=["checkpoint_eval/normalized_constraint_violation"],
        title="Learning Curves",
        sub_titles=["Average Normalized Constraint Violations"],
        xlabel="Training Time (s)",
        ylabels=["Constraint Violations"],
        window=None,
        x_num_max=None,
        x_rescale_factor=dt,
        num_std=0,
        use_median_quantile=True,
        cols_per_row=3
    )

    # Save stats to CSV.
    save_stats_to_csv(
        scalar_stats,
        algo_name_map,
        scalar_name_map,
        csv_path=os.path.join(config.plot_dir, "constraint_performance.csv")
    )
    print("Constraint plotting done.")

##############################################################################################
#############################   Mains   ######################################################
##############################################################################################

MAIN_FUNCS = {
    "test_policy": test_policy,
    "test_from_checkpoints": test_from_checkpoints,
    "plot_performance": plot_performance,
    "plot_constraint": plot_constraint,
}

if __name__ == "__main__":
    # Make config.
    fac = ConfigFactory()
    fac.add_argument("--func", type=str, default="test", help="main function to run.")
    fac.add_argument("--thread", type=int, default=0, help="number of threads to use (set by torch).")
    fac.add_argument("--render", action="store_true", help="if to render in policy test.")
    fac.add_argument("--verbose", action="store_true", help="if to print states & actions in policy test.")
    fac.add_argument("--set_test_seed", action="store_true", help="if to set seed when testing policy.")
    fac.add_argument("--eval_output_dir", type=str, help="folder path to save evaluation results.")
    fac.add_argument("--eval_output_path", type=str, default="test_results.pkl", help="file path to save evaluation results.")

    # testing args
    fac.add_argument("--restore_model_file", type=str, default="model_latest.pt", help="file name to restore the model.")
    fac.add_argument("--fix_env_init", action="store_true", help="if to test policy using env with fixed initial conditions.")
    fac.add_argument("--max_episode_plot", type=int, default=1, help="max number of test episode to plot.")
    fac.add_argument("--checkpoint_dir", type=str, default="checkpoints", help="folder where training checkpoints are saved.")
    fac.add_argument("--set_test_seed_as_training_eval", action="store_true", help="if to set seed same as eval_env during training.")
    fac.add_argument("--checkpoint_eval_dir", type=str, default="checkpoint_eval", help="where to save csv logs evaluated on checkpoints")

    # plotting args
    fac.add_argument("--plot_dir", type=str, help="folder path to save the plots.")

    config = fac.merge()
    # System settings.
    if config.thread > 0:
        # E.g. set single thread for less context switching
        torch.set_num_threads(config.thread)
    # Execute.
    func = MAIN_FUNCS.get(config.func, None)
    if func is None:
        raise Exception("Main function {} not supported.".format(config.func))
    func(config)


from sklearn.base import BaseEstimator, TransformerMixin
import pandas as pd
import numpy as np
from typing import List

class FormatMissingData(BaseEstimator, TransformerMixin):
    """Modify missing data values as required by model sklearn Pipeline."""

    def __init__(
        self,
        skip_inputs: List[str] = [],
    ):
        """
        Modify missing data values as required by model sklearn Pipeline.

        Parameters
        ----------
        skip_inputs : List[str], default []
            list of features whose missing values will
            not be adjusted
            if [], then *all* inputs are formatted

        Returns
        -------
        None.
        """
        self.skip_inputs = skip_inputs

    def fit(
        self,
        X=None,
        y=None,
    ):
        """
        Fit.

        Does nothing.

        Parameters
        ----------
        X : default None
            features data set
        y : default None
            target data set

        Returns
        -------
        self : FormatMissingData class
        """
        return self

    def transform(
        self,
        data: pd.DataFrame,
    ) -> pd.DataFrame:
        """
        Transform.

        Convert None or '' to np.nan.

        Parameters
        ----------
        data : pd.DataFrame
            data with missing values to be adjusted

        Returns
        -------
        data : pd.DataFrame
            same as what was passed in, but with missing data adjustments
        """
        for feature in data.columns:
            if feature not in self.skip_inputs:
                if np.issubdtype(data[feature].dtype, np.number):
                    data[feature].replace([None], [np.nan], inplace=True)
                else:
                    idx = data[feature].isin(['', None])
                    data.loc[idx, feature] = None

        return data


import webapp2
import jinja2
import os
import json
import logging
import datetime
import csv
from StringIO import StringIO
from urllib import quote, urlencode
from google.appengine.api import urlfetch

from models import TrackedUser, Followers

import headers

jinja_environment = jinja2.Environment(
    loader=jinja2.FileSystemLoader(os.path.join(os.path.dirname(__file__), "templates")))

class MainPage(webapp2.RequestHandler):
	def get(self):
		template = jinja_environment.get_template('index.html')

		template_values = {}

		self.response.out.write(template.render(template_values))

class TrackedUserList(webapp2.RequestHandler):
	def get(self):
		template = jinja_environment.get_template('user-listing.html')

		template_values = {'tracked_users' : TrackedUser.query().order(TrackedUser.username)}

		self.response.out.write(template.render(template_values))

class UserDetails(webapp2.RequestHandler):
	def get(self, username):
		template = jinja_environment.get_template('user-data.html')

		now = datetime.datetime.now()
		two_days_ago = now - datetime.timedelta(days=2)

		follower_data = [f for f in Followers.query(Followers.username == username, Followers.when >= two_days_ago).order(-Followers.when)]

		followers = [f.followers for f in follower_data]

		follower_change = max_followers = min_followers = 0
		if len(follower_data) > 1:
			follower_change = followers[0] - followers[-1]
			max_followers = max(followers)
			min_followers = min(followers)

		template_values = {'username' : username,
			'follower_data' : follower_data,
			'follower_change' : follower_change,
			'max_followers' : max_followers,
			'min_followers' : min_followers,}

		self.response.out.write(template.render(template_values))

class CsvDump(webapp2.RequestHandler):
	def get(self, username):
		follower_data = [f for f in Followers.query(Followers.username == username).order(-Followers.when)]

		headers.csv(self.response)
		buffer = StringIO()
		csv_writer = csv.writer(buffer)
		for f in follower_data:
			csv_writer.writerow([f.when.strftime("%Y-%m-%d %H:%M:%S"), f.followers])

		self.response.out.write(buffer.getvalue())

app = webapp2.WSGIApplication([('/', MainPage),
	('/tracked-users', TrackedUserList),
	('/user/(.*)/csv', CsvDump),
	('/user/(.*)', UserDetails)],
                              debug=True)


"""https://web3py.readthedocs.io/en/stable/"""
from typing import Union, Any

from eth_typing import URI
from web3.contract import Contract
from web3.types import Wei, Address, HexStr, TxParams

from .reader import Reader

class Transactor(Reader):
    def __init__(self, provider: Union[URI, str], timeout: int = 60) -> None:
        super().__init__(provider, timeout)

    def launch_transaction(
        self,
        private_key: Union[HexStr, str],
        to: Union[Address, str],
        value: Wei = 0,
        max_price: Wei = None,
        nonce: int = None,
    ) -> HexStr:
        sender = self.get_address(private_key)
        params = self.get_params(sender, max_price, nonce, value)
        transaction = {**params, "to": self.to_checksum(to)}
        transaction["gas"] = self.web3.eth.estimateGas(transaction)
        return self.send_raw(private_key, transaction)

    def launch_function(
        self,
        instance: Contract,
        private_key: Union[HexStr, str],
        function_name: str,
        *args: Any,
        value: Wei = 0,
        max_price: Wei = None,
        nonce: int = None
    ) -> HexStr:
        sender = self.get_address(private_key)
        params = self.get_params(sender, max_price, nonce, value)
        transaction = instance.functions.__dict__[function_name](
            *args
        ).buildTransaction(params)
        return self.send_raw(private_key, transaction)

    def send_raw(
        self, private_key: Union[HexStr, str], transaction: TxParams
    ) -> HexStr:
        raw_tx = self.sign_transaction(private_key, transaction)
        return self.to_hex(self.web3.eth.sendRawTransaction(raw_tx))


import boto3
import json
import logging
import datetime
from elasticsearch import Elasticsearch, RequestsHttpConnection
import uuid
from requests_aws4auth import AWS4Auth
import requests

logger = logging.getLogger()
logger.setLevel('ERROR')

ELASTIC_HOST = 'https://vpc-photos-cr73giiqwxko7a2t22rzqu44rq.us-east-1.es.amazonaws.com'
ELASTIC_PORT = 9200
ELASTIC_REGION = 'us-east-1'
ELASTIC_SERVICE = 'es'

credentials = boto3.Session().get_credentials()

awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, 'us-east-1', 'es', session_token=credentials.token)

class BaseObject(object):
    @classmethod
    def from_dict(cls, d):
        raise NotImplementedError('Not implement yet')

    @classmethod
    def from_json_str(cls, s):
        d = json.loads(s)
        return cls.from_dict(d)

    def to_dict(self):
        raise NotImplementedError('Not implement yet')

    def to_json_str(self):
        d = self.to_dict()
        return json.dumps(d)

class SearchResponse(BaseObject):
    def __init__(self, results):
        assert isinstance(results, list)
        for r in results:
            assert isinstance(r, Photo)
        self.results = results

    @classmethod
    def from_dict(cls, d):
        c = cls(results=[])
        for o in d['results']:
            c.add_result(Photo.from_dict(o))

    def add_result(self, photo):
        assert isinstance(photo, Photo)
        self.results.append(photo)

    def to_dict(self):
        return {'results': [o.to_dict() for o in self.results]}

class Photo(BaseObject):
    def __init__(self, url, labels):
        assert isinstance(url, str)
        assert isinstance(labels, list)
        for l in labels:
            assert isinstance(l, str)

        self.url = url
        self.labels = labels

    @classmethod
    def from_dict(cls, d):
        return cls(int(d['url']), d['labels'])

    def to_dict(self):
        return {
            'url': self.url,
            'labels': self.labels
        }

class Error(BaseObject):
    def __init__(self, code, message):
        assert isinstance(code, int)
        assert isinstance(message, str)
        self.code = code
        self.message = message

    @classmethod
    def from_dict(cls, d):
        return cls(int(d['code']), d['message'])

    def to_dict(self):
        return {
            'code': self.code,
            'message': self.message
        }

def build_s3_address(bucket, _id):
    # https://s3.amazonaws.com/cloudhw3bucket/201926383_22de827db0.jpg
    base = 'https://s3.amazonaws.com/'
    return base + bucket + '/' + _id

def build_response(status, body):
    return {
        'statusCode': status,
        'body': body
    }

def handler(event, context):
    logger.error('trigger lambda')

    try:
        lex_client = boto3.client('lex-runtime')

    except Exception:
        err = Error(-1, 'Something wrong with client.')
        return build_response('500', err.to_json_str())

    try:
        query = event['queryStringParameters']['q']
        query = query.replace('+', ' ')
    except KeyError as e:
        err = Error(-1, str(e))
        return build_response('500', err.to_json_str())

    logger.error('to lex')
    try:
        lex_output = lex_client.post_text(
            botName='getObjectBot',
            botAlias='getObject',
            userId=str(uuid.uuid4()),
            sessionAttributes={},
            requestAttributes={},
            inputText=query
        )
    except Exception as e:
        err = Error(-1, str(e))
        return build_response('500', err.to_json_str())

    try:
        slots = lex_output['slots']
        obj_a = slots['obj_a']
        obj_b = slots['obj_b']

    except KeyError as e:
        err = Error(-1, str(e))
        return build_response('500', err.to_json_str())

    logger.error(str(obj_a))
    logger.error(str(obj_b))

    # try:
    #     es_client = Elasticsearch(
    #         hosts=[{'host': ELASTIC_HOST}],
    #         # hosts=[{'host': ELASTIC_HOST, 'port': ELASTIC_PORT}],
    #         http_auth=awsauth,
    #         use_ssl=True,
    #         verify_certs=True,
    #         connection_class=RequestsHttpConnection
    #     )
    # except Exception as e:
    #     err = Error(-1, 'Something wrong with es client.')
    #     return err.to_json_str()

    labels = [obj_a, obj_b] if obj_b is not None else [obj_a]

    esq = {
        'size': 100,
        'query': {
            'bool': {
                "must": [
                    {"terms": {"labels": labels}}
                ]
            }
        }
    }
    logger.error('to es')

    index = 'photos'
    type = 'lambda-type'
    url1 = ELASTIC_HOST + '/' + index + '/_search'
    headers = {"Content-Type": "application/json"}

    res = requests.get(url1, auth=awsauth, headers=headers, data=json.dumps(esq))
    print(res.json())
    try:
        hits = res.json()['hits']['hits']
        logger.error(str(res.json()['hits']))
    except KeyError:
        hits = []

    # build response
    print(hits)
    rps = SearchResponse([])
    for h in hits:
        new_photo = Photo(build_s3_address(h['_source']['bucket'], h['_source']['objectKey']), h['_source']['labels'])
        rps.add_result(new_photo)
    logger.error(rps.to_json_str())

    return build_response('200', rps.to_json_str())


from fastapi import FastAPI
from fastapi.openapi.utils import get_openapi

from starlette.middleware.cors import CORSMiddleware
from app.core.config import ALLOWED_HOSTS, PROJECT_NAME

from app.api.endpoints import router as api_router

# create the app
app = FastAPI(title=PROJECT_NAME)

if not ALLOWED_HOSTS:
    ALLOWED_HOSTS = ['*']

app.add_middleware(
    CORSMiddleware,
    allow_origins=ALLOWED_HOSTS,
    allow_credentials=True,
    allow_methods=['*'],
    allow_headers=['*'],
)

# add endpoints to app
app.include_router(api_router)

# customize openapi documentation
def custom_openapi():
    """Customize openapi documentation page."""
    if app.openapi_schema:
        return app.openapi_schema
    openapi_schema = get_openapi(
        title="pollination.cloud",
        version="0.0.1",
        description="This page only documents %s endpoints."
        " See the link below for full API documentation." % PROJECT_NAME,
        routes=app.routes,
    )
    openapi_schema["info"]["x-logo"] = {
        "url": "https://user-images.githubusercontent.com/2915573/53690998-7239bb00-3d43-11e9-85b1-d9ac9d140c0f.png"
    }

    openapi_schema["info"]["license"] = {
        "name": "AGPL",
        "url": "https://www.gnu.org/licenses/agpl-3.0.en.html"
    }

    openapi_schema["externalDocs"] = {
        "description": "Pollination Cloud full API documentation.",
        "url": "https://pollination.cloud/api"
    }

    # openapi_schema["servers"] =  [{"url": "http://api.pollination.cloud"}]

    app.openapi_schema = openapi_schema
    return app.openapi_schema

app.openapi = custom_openapi


import cv2
import sklearn
import os
import numpy as np
import random
import skimage.transform as sktransform
from keras.preprocessing.image import random_shift

# 0.2,0.125

# def augment_brightness_camera_images(image):
#     image1 = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)
#     image1 = np.array(image1, dtype=np.float64)
#     if random.random() <= 0.5:
#         random_bright = 0.5 + np.random.uniform()*0.5
#     else:
#         random_bright = 1
#     image1[:, :, 2] = image1[:, :, 2] * random_bright
#     image1[:, :, 2][image1[:, :, 2] > 255] = 255
#     image1 = np.array(image1, dtype=np.uint8)
#     image1 = cv2.cvtColor(image1, cv2.COLOR_HSV2RGB)
#     return image1

def augment_brightness_camera_images(image):
    image1 = cv2.cvtColor(image,cv2.COLOR_RGB2HSV)
    image1 = np.array(image1, dtype = np.float64)
    random_bright = .5+np.random.uniform()
    image1[:,:,2] = image1[:,:,2]*random_bright
    image1[:,:,2][image1[:,:,2]>255]  = 255
    image1 = np.array(image1, dtype = np.uint8)
    image1 = cv2.cvtColor(image1,cv2.COLOR_HSV2RGB)
    return image1

def preprocess(image, top_offset=20, bottom_offset=20):
    """
    Applies preprocessing pipeline to an image: crops `top_offset` and `bottom_offset`
    portions of image, resizes to 32x128 px and scales pixel values to [0, 1].
    """
    input_size = image.shape
    height = input_size[0]
    crop_img = image[top_offset:height - bottom_offset]
    #final_img = augment_brightness_camera_images(crop_img)
    final_img = crop_img
    return final_img

cameras = ["left", "center", "right"]

def generator(df_samples, datafolder_path, augument=False, batch_size=32, opname=""):
    print("Start of Epoch")
    num_samples = len(df_samples)
    print("Num Samples:", num_samples)
    num_cameras = len(cameras)
    print(opname)

    # Reset offset to zero at begin of epoch
    offset = 0

    # Create Histogram
    df = df_samples
    num_bins = 200  # N of bins
    bin_list = [None] * num_bins
    start = 0
    i = 0
    for end in np.linspace(0, 1, num=num_bins):
        df_range = np.array(df[(np.absolute(df.steering) >= start) & (
            np.absolute(df.steering) < end)].index)
        # print(end)
        bin_list[i] = df_range
        #print("Index ",i, " :")
        # print(df_range)
        start = end
        i += 1
    final_bin_list = bin_list
    bin_cnt = num_bins*[0]

    while 1:  # Loop forever so the generator never terminates
        # Get the Batch
        j = 0
        start_index = 0
        table_indices = []
        while j < batch_size:
            # get row index
            #start_index = np.random.randint(0,num_bins/4)
            start_index = 0
            row_index = np.random.randint(start_index, num_bins)
            num_vals_bin = len(final_bin_list[row_index])
            if num_vals_bin > 0:
                col_index = np.random.randint(num_vals_bin)
                table_index = final_bin_list[row_index][col_index]
                j += 1
                table_indices.append(table_index)
                bin_cnt[row_index] +=1
        batch_samples = df.loc[table_indices]
        #print(np.max(bin_cnt))
        images = []
        angles = []
        iter = 0
        for index, batch_sample in batch_samples.iterrows():
            #name = datafolder_path+'/'+batch_sample['center']
            # table_index = (index+offset)%num_samples #Cicular Buffer Index over the rows of the table
            #batch_sample = df_samples[i]
            steer_angle = float(batch_sample['steering'])

            # Pick random camera : To teach how to steer from left to right, vice-versa
            # After adding this, the car was driving fine but keep ending
            # up in lake
            if augument:
                rand_index = random.randint(0, num_cameras - 1)
                random_camera = cameras[rand_index]
                if random_camera == "left":
                    steer_angle += 0.25
                elif random_camera == "right":
                    steer_angle -= 0.25 #Best is 0.25
            else:
                random_camera = "center"

            # Read file from folder
            name = os.path.join(datafolder_path, batch_sample[random_camera].strip())
            img = cv2.imread(name)
            correct_img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

            if type(img) == None:
                print("Error reading file, check file path")
                break

            # Make alternate image steer pos and negative (Balance positive and
            # negative angles)
            if (iter % 2 == 0 and steer_angle < 0) or (iter % 2 != 0 and steer_angle > 0):
                flip_img = cv2.flip(correct_img, 1)
                steer_angle = steer_angle * -1.0 # invert the steering angle
            else:
                flip_img = correct_img

            # Crop the image
            #v_delta = .05 if augument else 0
            crop_img = preprocess(flip_img, top_offset=75, bottom_offset=15)
            #crop_img = flip_img
            #bright_img = augment_brightness_camera_images(crop_img)

            if augument and random.random() <= 0.45:
                shift_img = random_shift(crop_img, 0, 0.2, 0, 1, 2)
                #bright_img = augment_brightness_camera_images(shift_img)
                bright_img = shift_img
            else:
                bright_img = crop_img

            if augument:
                bright_img = augment_brightness_camera_images(bright_img)

            image = bright_img

            if augument:
            # Add random shadow as a vertical slice of image
                h, w = image.shape[0], image.shape[1]
                [x1, x2] = np.random.choice(w, 2, replace=False)
                k = h / (x2 - x1)
                b = - k * x1
                for i in range(h):
                    c = int((i - b) / k)
                    image[i, :c, :] = (image[i, :c, :] * .5).astype(np.int32)

            final_img = image
            #steer_angle += 0.1*(np.random.rand()-0.5)
            # Accumulate the data
            images.append(final_img)
            angles.append(steer_angle)
            iter +=1
            if offset == 0 and iter == 0:
                cv2.imwrite("img0.jpg",final_img)

        # trim image to only see section with road
        #X_train = X_train[:,80:,:,:]
        X_train = np.array(images)
        y_train = np.array(angles)

        # Append data to csv file
        if opname == "train":
            filename = "train_log.csv"
            f = open(filename, "ab")
            np.savetxt(f, y_train)
            f.close()

        # print(y_train)
        #print("X shape:",X_train.shape)
        # return sklearn.utils.shuffle(X_train, y_train) #for testing
        # purposes using test_data_gen.py
        yield sklearn.utils.shuffle(X_train, y_train)
        offset += batch_size

    # return sklearn.utils.shuffle(X_train, y_train)


import asyncio

async def square(num):
    await asyncio.sleep(num)
    # print(f"tasks {num} compute value is {num * num}")
    return num

async def main():
    tasks = []
    for i in range(10, 0, -1):
        t1 = asyncio.create_task(square(i))
        tasks.append(t1)

    results = await asyncio.gather(*tasks)
    for result in results:
        print(result)

asyncio.run(main())


"""
Experiment metadata parsing and validation.
"""
import os.path as osp
import json
from collections import defaultdict
from lrgasp import LrgaspException, gopen, iter_to_str, existing_datafile_name
from lrgasp.objDict import ObjDict
from lrgasp.defs import Repository, Species, Challenge, DataCategory, Sample, LibraryPrep, Platform, EXPERIMENT_JSON
from lrgasp.defs import validate_symbolic_ident, is_simulation, challenge_desc, get_challenge_samples
from lrgasp.defs import MODELS_GTF, READ_MODEL_MAP_TSV, DE_NOVO_RNA_FASTA, EXPRESSION_TSV
from lrgasp.metadata_validate import Field, check_from_defs, validate_url
from lrgasp.data_sets import get_lrgasp_rna_seq_metadata, get_run_type, run_md_desc, file_md_desc

fld_notes = Field("notes", allow_empty=True, optional=True)

experiment_fields = (
    Field("experiment_id", validator=validate_symbolic_ident),
    Field("challenge_id", Challenge),
    Field("description"),
    Field("species", Species),
    Field("data_category", DataCategory),
    Field("samples", list, element_dtype=Sample),
    Field("library_preps", list, element_dtype=LibraryPrep),
    Field("platforms", list, element_dtype=Platform),
    Field("libraries", list, element_dtype=str, validator=validate_symbolic_ident),
    Field("extra_libraries", list, optional=True, allow_empty=True, element_dtype=dict),
    Field("software", list, element_dtype=dict),
    fld_notes,
)

extra_libraries_fields = (
    Field("acc"),
    Field("repository", Repository),
    fld_notes,
)

experiment_software_fields = (
    Field("name"),
    Field("version"),
    Field("description", optional=True),
    Field("url", validator=validate_url),
    Field("config"),
    fld_notes,
)

valid_file_content = frozenset(["reads", "subreads", "nanopore_signal", "R2C2_subreads"])

def get_extra_libraries(experiment_md):
    """get the extra_libraries field, or empty if not specified"""
    return experiment_md.get("extra_libraries", ())

def find_dups(lst):
    dups = []
    found = set()
    for v in lst:
        if (v in found) and (v not in dups):
            dups.append(v)
        found.add(v)
    return sorted(dups)

def get_libraries_file_metadata(rna_seq_md, experiment_md):
    """collect metadata for all library files in experiment"""
    # exception if unknown acc
    return [rna_seq_md.get_file_by_acc(file_acc)
            for file_acc in experiment_md.libraries]

def library_validate(experiment_md, rna_seq_md, file_md):
    run_md = rna_seq_md.get_run_by_acc(file_md.run_acc)
    if run_md.species != experiment_md.species:
        raise LrgaspException(f"LRGASP RNA-Seq library '{run_md_desc(run_md)}' file '{file_md_desc(file_md)}' is for species '{run_md.species}' while experiment '{experiment_md.experiment_id}' specifies species as '{experiment_md.species}'")

    # this should never happen, as the data matrix should be restrict to types we allow
    if file_md.output_type not in valid_file_content:
        raise LrgaspException(f"File '{file_md_desc(file_md)}' of output_type '{file_md.output_type}' not support for LRGASP, "
                              "valid types are {}; please contact LRGASP project if this type of file is needed".format(", ".join(sorted(valid_file_content))))

def get_file_run_type(rna_seq_md, file_md):
    return get_run_type(rna_seq_md.get_run_by_acc(file_md.run_acc))

def get_run_types(rna_seq_md, file_mds):
    "group into sets of long and short RunType,"
    long_run_types = set()
    short_run_types = set()
    for file_md in file_mds:
        rt = get_file_run_type(rna_seq_md, file_md)
        if rt.platform == Platform.Illumina:
            short_run_types.add(rt)
        else:
            long_run_types.add(rt)
    return long_run_types, short_run_types

def get_run_type_descs(rna_seq_md, file_mds):
    "produce list of string descriptions of file acc to run types for error messages"

    def _mk_desc(file_md, done):
        done.add(file_md.file_acc)
        if file_md.get("paired_with") is None:
            acc_desc = file_md.file_acc
        else:
            acc_desc = file_md.file_acc + ", " + file_md.paired_with
            done.add(file_md.paired_with)
        rt = get_file_run_type(rna_seq_md, file_md)
        return f"{acc_desc}: {rt}"

    descs = []
    done = set()  # to avoid duplicating due to paired_with
    for file_md in file_mds:
        if file_md.file_acc not in done:
            descs.append(_mk_desc(file_md, done))
    return descs

def _validate_sample_compat(rna_seq_md, experiment_md, run_type):
    valid_samples = get_challenge_samples(experiment_md.challenge_id)
    if run_type.sample not in valid_samples:
        raise LrgaspException(f"sample '{run_type.sample}' is not valid for challenge '{experiment_md.challenge_id}',"
                              " expected one of {}".format(iter_to_str(valid_samples)))

def _validate_samples_compat(rna_seq_md, experiment_md, run_types):
    """check if samples uses are compatible with challenge"""
    for run_type in run_types:
        _validate_sample_compat(rna_seq_md, experiment_md, run_type)

def _validate_data_category_compat(rna_seq_md, experiment_md, long_run_types, short_run_types, file_mds):  # noqa: C901
    """compatibility between data category and libraries in an experiments;"""
    data_category = experiment_md.data_category

    def _run_type_err_msg():
        return ("from specified library files:\n    " +
                "\n    ". join(get_run_type_descs(rna_seq_md, file_mds)))

    def _validate_short_only():
        if len(long_run_types) > 0:
            raise LrgaspException(f"{data_category} experiments must not use long-read platforms, found {len(long_run_types)}, " + _run_type_err_msg())
        if len(short_run_types) != 1:
            raise LrgaspException(f"{data_category} experiments must use one and only one short-read library/platform, found {len(short_run_types)}, " + _run_type_err_msg())

    def _validate_long_only():
        if len(long_run_types) != 1:
            raise LrgaspException(f"{data_category} experiments must use one and only one long-read library/platform, found {len(long_run_types)}, " + _run_type_err_msg())
        if len(short_run_types) != 0:
            raise LrgaspException(f"{data_category} experiments must not use short-read platforms, " + _run_type_err_msg())

    def _validate_long_short():
        if len(long_run_types) != 1:
            raise LrgaspException(f"{data_category} experiments must use one and only one long-read library/platform, found {len(long_run_types)}, " + _run_type_err_msg())
        if len(short_run_types) != 1:
            raise LrgaspException(f"{data_category} experiments must use one and only one short-read library/platform, found {len(short_run_types)}, " + _run_type_err_msg())
        if list(short_run_types)[0].sample != list(long_run_types)[0].sample:
            raise LrgaspException(f"{data_category} experiments must use the same sample for long and short libraries, got " +
                                  _run_type_err_msg())

    def _validate_long_genome():
        if experiment_md.challenge_id != Challenge.iso_detect_de_novo:
            raise LrgaspException(f"{data_category} only allowed for {challenge_desc(Challenge.iso_detect_de_novo)}")
        _validate_long_only()

    def _validate_freestyle():
        if len(experiment_md.libraries) == 0:
            raise LrgaspException(f"{data_category} experiments must use some LRGASP RNA-Seq libraries")
        # simulation not allowed
        for file_md in file_mds:
            if is_simulation(rna_seq_md.get_run_by_file_acc(file_md.file_acc).sample):
                raise LrgaspException(f"{data_category} experiments may not use simulation data '{file_md_desc(file_md)}'")

    if data_category == DataCategory.short_only:
        _validate_short_only()
    elif data_category == DataCategory.long_only:
        _validate_long_only()
    elif data_category == DataCategory.long_short:
        _validate_long_short()
    elif data_category == DataCategory.long_genome:
        _validate_long_genome()
    elif data_category == DataCategory.freestyle:
        _validate_freestyle()
    else:
        raise LrgaspException("bug")

def _validate_library_category_compat(rna_seq_md, experiment_md, file_mds, run_types):
    def _match_err(fld_name, fld_value, expect_value):
        raise LrgaspException(f"experiment '{experiment_md.experiment_id}' field '{fld_name}', '{iter_to_str(fld_value)}', must match"
                              f" '{fld_name}' for all library files, got '{iter_to_str(expect_value)}', "
                              "from specified library files:\n    " +
                              "\n    ". join(get_run_type_descs(rna_seq_md, file_mds)))

    run_samples = set([rt.sample for rt in run_types])
    if set(run_samples) != set(experiment_md.samples):
        _match_err("samples", experiment_md.samples, run_samples)
    run_library_preps = set([rt.library_prep for rt in run_types])
    if set(run_library_preps) != set(experiment_md.library_preps):
        _match_err("library_preps", experiment_md.library_preps, run_library_preps)
    run_platforms = set([rt.platform for rt in run_types])
    if run_platforms != set(experiment_md.platforms):
        _match_err("platform", experiment_md.platforms, run_platforms)

def _validate_library_compat(rna_seq_md, experiment_md, file_mds):
    """compatibility between challenge, data category and libraries in an experiments;"""
    long_run_types, short_run_types = get_run_types(rna_seq_md, file_mds)
    run_types = long_run_types | short_run_types
    _validate_samples_compat(rna_seq_md, experiment_md, run_types)
    _validate_data_category_compat(rna_seq_md, experiment_md, long_run_types, short_run_types, file_mds)
    _validate_library_category_compat(rna_seq_md, experiment_md, file_mds, run_types)

def _validate_paired_end(rna_seq_md, file_mds):
    "make sure both paired ends are there"
    paired_file_mds = [fm for fm in file_mds if fm.paired_end is not None]
    paired_acc = frozenset([fm.file_acc for fm in paired_file_mds])
    for file_md in paired_file_mds:
        if file_md.paired_with not in paired_acc:
            raise LrgaspException(f"'{file_md_desc(file_md)}' is a paired-end experiment, however the paired file '{file_md.paired_with}' is not specified in experiment_md.libraries")

def get_runs_replicates(rna_seq_md, file_mds):
    "get dict of runs to set of replicates for all libraries"
    runs_replicates = defaultdict(set)
    for file_md in file_mds:
        runs_replicates[file_md.run_acc].add(file_md.biological_replicate_number)
    runs_replicates.default_factory = None
    return runs_replicates

def _validate_replicates(rna_seq_md, experiment_md, file_mds):
    """validate that all replicates are used in an experiment"""
    runs_replicates = get_runs_replicates(rna_seq_md, file_mds)

    def _get_run_file_desc(run_md):
        """generate list of possible files"""
        rep_file_descs = []
        for rep in run_md.replicates:
            for f in rep.files:
                rep_file_descs.append(f"{f.run_acc} {f.file_acc} {f.biological_replicate_number} {f.file_type}")
        return rep_file_descs

    def _check_run(run_acc, rep_set):
        run_md = rna_seq_md.get_run_by_acc(run_acc)
        if len(rep_set) != len(run_md.replicates):
            raise LrgaspException(f"experiment must use all replicates from run {run_acc}, available replicate files:\n    "
                                  + "\n    ".join(_get_run_file_desc(run_md)))

    for run_acc, rep_set in runs_replicates.items():
        _check_run(run_acc, rep_set)

def _validate_libraries(experiment_md):
    rna_seq_md = get_lrgasp_rna_seq_metadata()
    dups = find_dups(experiment_md.libraries)
    if len(dups) > 0:
        raise LrgaspException(f"duplicate accession in libraries: {dups}")

    # per library validation
    file_mds = get_libraries_file_metadata(rna_seq_md, experiment_md)
    for file_md in file_mds:
        library_validate(experiment_md, rna_seq_md, file_md)

    # cross-library file validations
    _validate_paired_end(rna_seq_md, file_mds)
    _validate_library_compat(rna_seq_md, experiment_md, file_mds)
    _validate_replicates(rna_seq_md, experiment_md, file_mds)

def _validate_extra_libraries(experiment_md):
    if experiment_md.data_category not in (DataCategory.long_short, DataCategory.freestyle):
        raise LrgaspException("experiment extra_libraries may only be specified for 'long_short' or 'freestyle' experiments")
    dups = find_dups([el.acc for el in experiment_md.extra_libraries])
    if len(dups) > 0:
        raise LrgaspException(f"duplicate accession in extra libraries: {dups}")
    for ilib in range(len(experiment_md.extra_libraries)):
        check_from_defs(f"experiment_md.extra_libraries[{ilib}]",
                        extra_libraries_fields, experiment_md.extra_libraries[ilib])

def _validate_software(experiment_md):
    for isoft in range(len(experiment_md.software)):
        check_from_defs(f"experiment_md.software[{isoft}]",
                        experiment_software_fields, experiment_md.software[isoft])

def experiment_validate(experiment_md):
    _validate_libraries(experiment_md)
    if len(get_extra_libraries(experiment_md)) > 0:
        _validate_extra_libraries(experiment_md)
    _validate_software(experiment_md)

def load(experiment_json):
    """load and validate experiment metadata"""
    try:
        with gopen(experiment_json) as fh:
            experiment_md = json.load(fh, object_pairs_hook=ObjDict)
    except json.decoder.JSONDecodeError as ex:
        raise LrgaspException(f"parse of experiment metadata (JSON) failed: {experiment_json}") from ex
    try:
        check_from_defs("experiment", experiment_fields, experiment_md)
        # add non-serialized fields after field check
        experiment_md.experiment_dir = None
        experiment_md.experiment_json = experiment_json
        experiment_validate(experiment_md)
    except LrgaspException as ex:
        raise LrgaspException(f"validation of experiment metadata failed: {experiment_json}") from ex

    return experiment_md

def load_from_entry(entry_md, experiment_id):
    """load and validation experiment metadata given a entry,
    add experiment_dir to metadata"""
    experiment_dir = osp.join(entry_md.entry_dir, experiment_id)
    experiment_json = osp.join(experiment_dir, EXPERIMENT_JSON)
    try:
        experiment_md = load(experiment_json)
        # add non-serialized fields after field check
        experiment_md.experiment_dir = osp.normpath(experiment_dir)
    except (LrgaspException, FileNotFoundError, ValueError) as ex:
        raise LrgaspException(f"error parse metadata for entry '{entry_md.entry_id}', experiment '{experiment_id}' (obtained from entry.json): {experiment_json}") from ex

    if experiment_md.experiment_id != osp.basename(experiment_dir):
        raise LrgaspException(f"experiment_id and directory name must be the same; '{experiment_md.experiment_id}' is in '{osp.basename(experiment_dir)}' ({experiment_dir})")

    return experiment_md

def get_models_gtf(experiment_md):
    """get path to models.gtf file or None if not valid for this experiment.
    Will not include .gz extension or check for existence."""
    if experiment_md.challenge_id in (Challenge.iso_detect_ref, Challenge.iso_quant):
        return existing_datafile_name(osp.join(experiment_md.experiment_dir, MODELS_GTF))
    else:
        return None

def get_read_model_map_tsv(experiment_md):
    """get path to read_model_map.gtf file or None if not valid for this
    experiment. Will not include .gz extension or check for existence."""
    if experiment_md.challenge_id in (Challenge.iso_detect_ref, Challenge.iso_detect_de_novo):
        return existing_datafile_name(osp.join(experiment_md.experiment_dir, READ_MODEL_MAP_TSV))
    else:
        return None

def get_rna_fasta(experiment_md):
    """get path to RNA fasta file or None if not valid for this
    experiment. Will not include .gz extension or check for existence."""
    if experiment_md.challenge_id is Challenge.iso_detect_de_novo:
        return existing_datafile_name(osp.join(experiment_md.experiment_dir, DE_NOVO_RNA_FASTA))
    else:
        return None

def get_expression_tsv(experiment_md):
    """get path to expression TSV file or None if not valid for this
    experiment. Will not include .gz extension or check for existence."""
    if experiment_md.challenge_id is Challenge.iso_quant:
        return existing_datafile_name(osp.join(experiment_md.experiment_dir, EXPRESSION_TSV))
    else:
        return None


#!/usr/bin/env python
# -*- encoding: utf-8

import gzip
import scipy.io as sio
from utils.utils import Utils

class GenericSparseDB(Utils):

   def init(self):
      self.data = sio.mmread(gzip.open(self._matrix_fn)).tolil()
      self.factors = self._load_pickle(self._factors_fn)
      self.fac_len = len(self.factors)
      self.col_names = self.factors + self._load_pickle(self._colnames_fn)
      assert self.data.shape[1] == len(self.col_names),\
                 'Mismatch between the number of columns: %s - %s.'\
                     % (self.data.shape[1], len(self.col_names))

   def reset(self):
      self.init()


import pandas as pd
from PyQt5 import QtGui
from PyQt5.QtCore import Qt
from PyQt5.QtWidgets import *
from pandas import DataFrame as DataframeObject
import Dataframe
from GraphMenu import GraphMenu
from GraphModel import GraphModel
from LoadedSheets import LoadedSheets
from PandasModel import PandasModel
import sys

# Main window creation
class MainWindow(QMainWindow):
    # Excel sheets
    sheetsDir: LoadedSheets
    # Tabs
    tabs: QTabWidget
    # Graph list
    graph_window: list[QWidget]
    # Dataframe list
    data: list[DataframeObject]
    # Graph menu
    menu: list[GraphMenu]

    def __init__(self, parent=None):
        super().__init__(parent)

        # Initialization
        container = QWidget()
        self.setCentralWidget(container)
        layout = QGridLayout()
        container.setLayout(layout)

        self.sheetsDir = LoadedSheets(self)
        self.tabs = QTabWidget(self)
        self.data = []
        self.menu = []
        self.graph_window = []

        # Set functions for click events
        self.sheetsDir.loadedSheets.itemDoubleClicked.connect(self.loadSheet)
        self.sheetsDir.saveButton.clicked.connect(self.save)
        self.tabs.tabBarDoubleClicked.connect(self.del_tab)

        # Add widgets to window
        layout.addWidget(self.sheetsDir, 0, 0, 5, 2)
        layout.addWidget(self.tabs, 0, 2, 5, 7)

        # Window setup
        self.setWindowTitle("Simple Cell")
        self.setMinimumSize(1067, 600)
        self.setWindowIcon(QtGui.QIcon('logo.jpg'))
        self.show()

    # Add tab to window
    def add_tab(self, widget: QWidget, name):
        tab = QWidget()
        layout = QGridLayout()
        tab.setLayout(layout)

        layout.addWidget(widget, 0, 0)

        # Add graph menu to tab
        m = GraphMenu()
        m.graph_type.currentIndexChanged.connect(self.set_combobox)
        m.create.clicked.connect(self.create_graph)
        layout.addWidget(m, 1, 0)
        self.menu.append(m)

        # Add tab
        self.tabs.addTab(tab, name)
        self.tabs.setCurrentIndex(len(self.tabs) - 1)

    # Delete tab from double click
    def del_tab(self):
        if len(self.data) > 0:
            tab_index = self.tabs.currentIndex()
            self.tabs.removeTab(tab_index)
            del self.menu[tab_index]
            del self.data[tab_index]

    # Load sheet
    def loadSheet(self, file: QListWidgetItem):

        # If sheet doesn't already exist, load it
        if not self.tab_exists(file.text()):
            excel_sheet_path = self.sheetsDir.getPath(file)
            sheet = excel_sheet_path[excel_sheet_path.rfind(" - ") + 3:]
            path = excel_sheet_path[:excel_sheet_path.rfind(" - ")]

            # Create dataframe
            df = Dataframe.excel_to_dataframe(path, sheet)
            self.data.append(df)
            df_model = self.create_dataframe_model(df)
            self.add_tab(df_model, file.text())

            df_cols = list(df.columns)

            # Add dataframe columns to graph menu
            for col in df_cols:
                self.menu[len(self.menu) - 1].column1.addItem(col)
                self.menu[len(self.menu) - 1].column2.addItem(col)
        else:
            return

    # Check if tab already exists
    def tab_exists(self, name):
        tab_count = self.tabs.count()

        if tab_count > 0:
            for tab in range(0, self.tabs.count()):
                if self.tabs.tabText(tab) == name:
                    return True

            return False
        else:
            return False

    # Create dataframe model for table widget
    def create_dataframe_model(self, df: DataframeObject):
        model = PandasModel(df)
        view = QTableView()
        view.setModel(model)
        view.model().dataChanged.connect(self.changedData)

        return view

    # Create a graph
    def create_graph(self):
        # Get selected graph menu options
        tab_index = self.tabs.currentIndex()
        graph = GraphModel()
        selected_graph = self.menu[tab_index].graph_type.currentText()
        selected_col1 = self.menu[tab_index].column1.currentText()
        selected_col2 = self.menu[tab_index].column2.currentText()

        # Create line graph
        if selected_graph == "Line" and selected_col1 != "Select Column 1":
            try:
                graph.axes.plot(self.data[tab_index][selected_col1])
                self.create_graph_window(graph)
            except Exception:
                self.error_window()
                return

        # Create bar graph
        elif selected_graph == "Bar" and selected_col1 != "Select Column 1" and \
                selected_col2 != "Select Column 2":
            try:
                graph.axes.bar(self.data[tab_index][selected_col1], self.data[tab_index][selected_col2])
                self.create_graph_window(graph)
            except Exception:
                self.error_window()
                return
        # If options are not correctly selected, don't make graph
        else:
            return

    # Create new window for graph
    def create_graph_window(self, graph):

        # Create save button
        save = QPushButton("Save as PNG")

        layout = QGridLayout()
        layout.addWidget(save, 0, 0)
        layout.addWidget(graph, 1, 0)

        tab_index = self.tabs.currentIndex()
        selected_graph = self.menu[tab_index].graph_type.currentText()
        selected_col1 = self.menu[tab_index].column1.currentText()
        selected_col2 = self.menu[tab_index].column2.currentText()

        w = QWidget()
        w.setLayout(layout)
        w.setWindowIcon(QtGui.QIcon('logo.jpg'))

        # Set proper title
        if selected_graph == "Line":
            w.setWindowTitle("Line Graph [" + selected_col1 + "]")
        elif selected_graph == "Bar":
            w.setWindowTitle("Bar Graph [" + selected_col1 + ", " + selected_col2 + "]")

        # Get click event for save graph
        save.clicked.connect(self.save_graph)

        self.graph_window.append(w)
        self.graph_window[len(self.graph_window) - 1].show()
        self.graph_window[len(self.graph_window) - 1].setMinimumSize(w.size())

    # Save graph as png
    def save_graph(self):
        # Get graph and file name
        graph = self.graph_window[len(self.graph_window) - 1].findChild(GraphModel)
        file_name = self.graph_window[len(self.graph_window) - 1].windowTitle()

        tab_index = self.tabs.currentIndex()
        tab_title = self.tabs.tabText(tab_index)
        path = self.sheetsDir.getPath(fileName=tab_title)

        # Get correct path
        last_char = path[-1]
        while last_char != "/":
            path = path[:-1]
            last_char = path[-1]

        # Save as png
        graph.axes.figure.savefig(path + file_name + ".png")

    # Error window that displays when graph creation throws error
    def error_window(self):
        error = QLabel("Graph Error: Unable to create graph\n\nCheck for bad data")
        error.setAlignment(Qt.AlignCenter)

        layout = QVBoxLayout()
        layout.addWidget(error)

        w = QWidget()
        w.setLayout(layout)
        w.setWindowTitle("Graph Error")
        w.setFixedSize(300, 100)
        w.setWindowIcon(QtGui.QIcon('logo.jpg'))

        self.graph_window.append(w)
        self.graph_window[len(self.graph_window) - 1].show()

    # Enable or disable graph menu widgets accordingly
    def set_combobox(self):
        tab_index = self.tabs.currentIndex()
        selected = self.menu[tab_index].graph_type.currentText()

        if selected == "Line":
            self.menu[tab_index].column1.setEnabled(True)
            self.menu[tab_index].column2.setEnabled(False)
        elif selected == "Bar":
            self.menu[tab_index].column1.setEnabled(True)
            self.menu[tab_index].column2.setEnabled(True)
        else:
            self.menu[tab_index].column1.setEnabled(False)
            self.menu[tab_index].column2.setEnabled(False)

    def changedData(self, item):
        print(item.row())
        print(item.column())

    # Overwrite Excel file with edited data
    def save(self):
        tab_index = self.tabs.currentIndex()

        if tab_index >= 0:
            # Get path and file name
            tab_title = self.tabs.tabText(tab_index)
            path = self.sheetsDir.getPath(fileName=tab_title)
            filename = path[:path.rfind(" - ")]
            sheet = tab_title[tab_title.rfind(" - ") + 3:]

            # Write to file
            with pd.ExcelWriter(filename, engine='openpyxl', mode='a') as writer:
                workBook = writer.book
                try:
                    workBook.remove(workBook[sheet])
                except:
                    print("Worksheet does not exist")
                finally:
                    self.data[tab_index].to_excel(writer, sheet_name=sheet, index=False)

if __name__ == "__main__":
    app = QApplication(sys.argv)
    windowExample = MainWindow()
    sys.exit(app.exec_())


#!/usr/bin/env python3

"""
Unreal Engine 5: Plugin Manager
"""
import os
import json
import tkinter
from tkinter import ttk, messagebox
from ttkwidgets import CheckboxTreeview

script_directory = os.path.dirname(os.path.abspath(__file__))
uproject_files = [f for f in os.listdir(script_directory) if f.endswith('.uproject')]
if not uproject_files:
    messagebox.showerror("Error", "No .uproject found!")
    exit(1)

project_path = os.path.join(script_directory, uproject_files[0])

window = tkinter.Tk()
window.title("UE5 Plugin Manager")
window.minsize(320, 480)

tree = CheckboxTreeview(window)
tree.pack(expand=True, fill=tkinter.BOTH)

with open(project_path, "r") as f:
    uproject_json = json.load(f)

checkboxes = {}
plugins = uproject_json.get("Plugins", [])
for plugin in plugins:
    plugin_name = plugin.get("Name", "")
    checkboxes[plugin_name] = plugin.get("Enabled", False)

for plugin_name in sorted(checkboxes.keys(), reverse=True):
    enabled = checkboxes[plugin_name]
    tree.insert("", 0, iid=plugin_name, text=plugin_name, tags=("checked" if enabled else "unchecked"))

select_all_button = ttk.Button(window, text="All", command=tree.check_all)
select_all_button.pack(side=tkinter.LEFT, padx=5, pady=5)

deselect_all_button = ttk.Button(window, text="None", command=tree.uncheck_all)
deselect_all_button.pack(side=tkinter.LEFT, padx=0, pady=5)

def save_plugins():
    for item in tree.get_children(""):
        checkboxes[item] = tree.tag_has("checked", item)

    for plugin in plugins:
        plugin_name = plugin.get("Name", "")
        plugin["Enabled"] = checkboxes[plugin_name]

    with open(project_path, "w") as f:
        json.dump(uproject_json, f, indent=4)

save_button = ttk.Button(window, text="Save", command=save_plugins)
save_button.pack(side=tkinter.RIGHT, padx=5, pady=5)

window.mainloop()


# -*- coding: utf-8 -*-

import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from recommender.models import Movie

class ContentSimilarity:
    # MELHORIA FUTURA: SALVAR MOST SIMILAR NO BD E ATUALIZAR SOMENTE QDO NECESSÁRIO
    def content_similarity(self, movie_id):
        #transform the Movie QuerySet into a pandas DataFrame
        movies = pd.DataFrame(list(Movie.objects.all().values()))

        #model instantiation
        vectorizer = TfidfVectorizer(min_df=1)

        #tokenize and compute tf-idf weights
        tfidf = vectorizer.fit_transform(movies.movie_description)

        #compute the similarity between movies
        similar = cosine_similarity(tfidf)

        #take the top-5 most similar movies
        most_similar = similar.argsort()[:,::-1][:,:6]

        most_similar = pd.DataFrame(most_similar)

        list_similar_movies = []

        #make a list of top-k recommended movies
        for i in range (1,most_similar.shape[1]):
            sim_movie_id = most_similar.iloc[int(movie_id),i]
            list_similar_movies.append(Movie.objects.get(pk=sim_movie_id))
            # print ("Name: %s" %movies.title.iloc[most_similar[movie_id,i]])
            # print ("Cosine similarity score: %f \n" %similar[movie_id,most_similar[movie_id,i]])
        return list_similar_movies

    # def __call__(movie_id):
    #     print ('called')


'''
Created on Feb 25, 2010

'''
from math import pi as Pi, cos, sin, exp, sqrt as scalar_sqrt

from ibvpy.tmodel.mats2D.mats2D_eval import MATS2DEval
from numpy import \
     array, zeros, dot, \
     float_, \
     sign
from traits.api import \
     Array, Enum, \
     Event, provides, \
     Dict, Property, cached_property
import bmcs_utils.api as bu

#---------------------------------------------------------------------------
# Material time-step-evaluator for Scalar-Damage-Model
#---------------------------------------------------------------------------
class MATS2D5PlasticBond(MATS2DEval):
    '''
    Elastic Model.
    '''

    #---------------------------------------------------------------------------
    # Parameters of the numerical algorithm (integration)
    #---------------------------------------------------------------------------

    stress_state = Enum("plane_stress", "plane_strain")

    #---------------------------------------------------------------------------
    # Material parameters
    #---------------------------------------------------------------------------

    E_m = bu.Float(1.,  # 34e+3,
                 label="E_m",
                 desc="Young's Modulus",
                 auto_set=False)
    nu_m = bu.Float(0.2,
                 label='nu_m',
                 desc="Poison's ratio",
                 auto_set=False)

    E_f = bu.Float(1.,  # 34e+3,
                 label="E_f",
                 desc="Young's Modulus",
                 auto_set=False)
    nu_f = bu.Float(0.2,
                 label='nu_f',
                 desc="Poison's ratio",
                 auto_set=False)

    G = bu.Float(1.,  # 34e+3,
                 label="G",
                 desc="Shear Modulus",
                 auto_set=False)

    sigma_y = bu.Float(.5,  # 34e+3,
                label="s_y",
                desc="Yield stress",
                auto_set=False)

    K_bar = bu.Float(0.,  # 34e+3,
                label="K",
                desc="isotropic hardening",
                auto_set=False)

    H_bar = bu.Float(0.,  # 34e+3,
                   label="H",
                   desc="kinematic hardening",
                   auto_set=False)

    D_el = Property(Array(float), depends_on='E_f, nu_f,E_m,nu_f,G, stress_state')

    @cached_property
    def _get_D_el(self):
        if self.stress_state == "plane_stress":
            return self._get_D_plane_stress()
        else:
            return self._get_D_plane_strain()

    # This event can be used by the clients to trigger an action upon
    # the completed reconfiguration of the material model
    #
    changed = Event

    #---------------------------------------------------------------------------------------------
    # View specification
    #---------------------------------------------------------------------------------------------

    view_traits = bu.View(
        bu.Item('E_m'),
        bu.Item('nu_m'),
        bu.Item('E_f'),
        bu.Item('nu_f'),
        bu.Item('G')
    )

    def get_corr_pred(self, eps_app_eng, tn1, eps_p_n, q_n, alpha_n):
        '''
        Corrector predictor computation.
        @param eps_app_eng input variable - engineering strain
        '''
        sigma = dot(self.D_el[:], eps_app_eng)

        # You print the stress you just computed and the value of the apparent E
        eps_n1 = float(eps_app_eng[6])  # hack for this particular case
        G = self.G

        eps_avg = eps_n1

        # print 'state array ', sctx.mats_state_array
        sigma_trial = self.G * (eps_n1 - eps_p_n)
        xi_trial = sigma_trial - q_n
        f_trial = abs(xi_trial) - (self.sigma_y + self.K_bar * alpha_n)
        # f_trial = -xi_trial - ( self.sigma_y + self.K_bar * alpha_n )

        sig_n1 = zeros((1,), dtype='float_')
        D_n1 = zeros((1, 1), dtype='float_')
        if f_trial <= 1e-8:
            sig_n1[0] = sigma_trial
            D_n1[0, 0] = G
        else:
            # print 'plastic'
            d_gamma = f_trial / (self.G + self.K_bar + self.H_bar)
            sig_n1[0] = sigma_trial - d_gamma * self.G * sign(xi_trial)
            D_n1[0, 0] = (self.G * (self.K_bar + self.H_bar)) / \
                            (self.G + self.K_bar + self.H_bar)
            # print 'stress ', sig_n1[0]
        sigma[6] = sig_n1[0]
        self.D_el[6, 6] = D_n1[0, 0]
        return  sigma, self.D_el

    def _get_D_plane_stress(self):
        E_m = self.E_m
        nu_m = self.nu_m
        E_f = self.E_f
        nu_f = self.nu_f
        G = self.G
        D_stress = zeros([8, 8])
        D_stress[0, 0] = E_m / (1.0 - nu_m * nu_m)
        D_stress[0, 1] = E_m / (1.0 - nu_m * nu_m) * nu_m
        D_stress[1, 0] = E_m / (1.0 - nu_m * nu_m) * nu_m
        D_stress[1, 1] = E_m / (1.0 - nu_m * nu_m)
        D_stress[2, 2] = E_m / (1.0 - nu_m * nu_m) * (1.0 / 2.0 - nu_m / 2.0)

        D_stress[3, 3] = E_f / (1.0 - nu_f * nu_f)
        D_stress[3, 4] = E_f / (1.0 - nu_f * nu_f) * nu_f
        D_stress[4, 3] = E_f / (1.0 - nu_f * nu_f) * nu_f
        D_stress[4, 4] = E_f / (1.0 - nu_f * nu_f)
        D_stress[5, 5] = E_f / (1.0 - nu_f * nu_f) * (1.0 / 2.0 - nu_f / 2.0)

        D_stress[6, 6] = G
        D_stress[7, 7] = G
        return D_stress

    def _get_D_plane_strain(self):
        # TODO: adapt to use arbitrary 2d model following the 1d5 bond
        E_m = self.E_m
        nu_m = self.nu_m
        E_f = self.E_f
        nu_f = self.nu_f
        G = self.G
        D_strain = zeros([8, 8])
        D_strain[0, 0] = E_m * (1.0 - nu_m) / (1.0 + nu_m) / (1.0 - 2.0 * nu_m)
        D_strain[0, 1] = E_m / (1.0 + nu_m) / (1.0 - 2.0 * nu_m) * nu_m
        D_strain[1, 0] = E_m / (1.0 + nu_m) / (1.0 - 2.0 * nu_m) * nu_m
        D_strain[1, 1] = E_m * (1.0 - nu_m) / (1.0 + nu_m) / (1.0 - 2.0 * nu_m)
        D_strain[2, 2] = E_m * (1.0 - nu_m) / (1.0 + nu_m) / (2.0 - 2.0 * nu_m)

        D_strain[3, 3] = E_f * (1.0 - nu_f) / (1.0 + nu_f) / (1.0 - 2.0 * nu_f)
        D_strain[3, 4] = E_f / (1.0 + nu_f) / (1.0 - 2.0 * nu_f) * nu_f
        D_strain[4, 3] = E_f / (1.0 + nu_f) / (1.0 - 2.0 * nu_f) * nu_f
        D_strain[4, 4] = E_f * (1.0 - nu_f) / (1.0 + nu_f) / (1.0 - 2.0 * nu_f)
        D_strain[5, 5] = E_f * (1.0 - nu_f) / (1.0 + nu_f) / (2.0 - 2.0 * nu_f)

        D_strain[6, 6] = G
        D_strain[7, 7] = G
        return D_strain

    #---------------------------------------------------------------------------------------------
    # Response trace evaluators
    #---------------------------------------------------------------------------------------------

    def get_sig_norm(self, sctx, eps_app_eng):
        sig_eng, D_mtx = self.get_corr_pred(sctx, eps_app_eng, 0, 0, 0)
        return array([ scalar_sqrt(sig_eng[0] ** 2 + sig_eng[1] ** 2) ])

    def get_eps_app_m(self, sctx, eps_app_eng):
        return self.map_eps_eng_to_mtx((eps_app_eng[:3]))

    def get_eps_app_f(self, sctx, eps_app_eng):
        return self.map_eps_eng_to_mtx((eps_app_eng[3:6]))

    def get_sig_app_m(self, sctx, eps_app_eng, *args, **kw):
        sig_eng, D_mtx = self.get_corr_pred(sctx, eps_app_eng, 0, 0, 0)
        return self.map_sig_eng_to_mtx((sig_eng[:3]))

    def get_sig_app_f(self, sctx, eps_app_eng, *args, **kw):
        sig_eng, D_mtx = self.get_corr_pred(sctx, eps_app_eng, 0, 0, 0)
        return self.map_sig_eng_to_mtx((sig_eng[3:6]))

    def get_sig_b(self, sctx, eps_app_eng, *args, **kw):
        sig_eng, D_mtx = self.get_corr_pred(sctx, eps_app_eng, 0, 0, 0)
        return array([[sig_eng[6], 0.], [0., sig_eng[7]]])

    # Declare and fill-in the rte_dict - it is used by the clients to
    # assemble all the available time-steppers.
    #
    rte_dict = Trait(Dict)

    def _rte_dict_default(self):
        return {
                'eps_app_f'  : self.get_eps_app_f,
                'eps_app_m'  : self.get_eps_app_m,
                'sig_app_f'  : self.get_sig_app_f,
                'sig_app_m'  : self.get_sig_app_m,
                'sig_norm'   : self.get_sig_norm,
                'sig_b'      : self.get_sig_b, }