#!/usr/bin/env python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
from __future__ import annotations

import json
import os
import sys
from ast import Import, ImportFrom, NodeVisitor, parse
from collections import defaultdict
from pathlib import Path
from typing import Any, List

import yaml
from rich.console import Console

console = Console(color_system="standard", width=200)

AIRFLOW_PROVIDERS_IMPORT_PREFIX = "airflow.providers."

AIRFLOW_SOURCES_ROOT = Path(__file__).parents[3].resolve()

AIRFLOW_PROVIDERS_DIR = AIRFLOW_SOURCES_ROOT / "airflow" / "providers"
AIRFLOW_TESTS_PROVIDERS_DIR = AIRFLOW_SOURCES_ROOT / "tests" / "providers"
AIRFLOW_SYSTEM_TESTS_PROVIDERS_DIR = AIRFLOW_SOURCES_ROOT / "system" / "tests" / "providers"

DEPENDENCIES_JSON_FILE_PATH = AIRFLOW_SOURCES_ROOT / "generated" / "provider_dependencies.json"

sys.path.insert(0, str(AIRFLOW_SOURCES_ROOT))  # make sure setup is imported from Airflow

warnings: list[str] = []
errors: list[str] = []

suspended_paths: list[str] = []

ALL_DEPENDENCIES: dict[str, dict[str, list[str]]] = defaultdict(lambda: defaultdict(list))

ALL_PROVIDERS: dict[str, dict[str, Any]] = defaultdict(lambda: defaultdict())
ALL_PROVIDER_FILES: list[Path] = []

# Allow AST to parse the files.
sys.path.append(str(AIRFLOW_SOURCES_ROOT))

class ImportFinder(NodeVisitor):
    """
    AST visitor that collects all imported names in its imports
    """

    def __init__(self) -> None:
        self.imports: list[str] = []
        self.handled_import_exception = List[str]
        self.tried_imports: list[str] = []

    def process_import(self, import_name: str) -> None:
        self.imports.append(import_name)

    def get_import_name_from_import_from(self, node: ImportFrom) -> list[str]:
        import_names: list[str] = []
        for alias in node.names:
            name = alias.name
            fullname = f"{node.module}.{name}" if node.module else name
            import_names.append(fullname)
        return import_names

    def visit_Import(self, node: Import):
        for alias in node.names:
            self.process_import(alias.name)

    def visit_ImportFrom(self, node: ImportFrom):
        if node.module == "__future__":
            return
        for fullname in self.get_import_name_from_import_from(node):
            self.process_import(fullname)

def find_all_providers_and_provider_files():
    for (root, _, filenames) in os.walk(AIRFLOW_PROVIDERS_DIR):
        for filename in filenames:
            if filename == "provider.yaml":
                provider_file = Path(root, filename)
                provider_name = str(provider_file.parent.relative_to(AIRFLOW_PROVIDERS_DIR)).replace(
                    os.sep, "."
                )
                provider_info = yaml.safe_load(provider_file.read_text())
                if not provider_info["suspended"]:
                    ALL_PROVIDERS[provider_name] = provider_info
                else:
                    suspended_paths.append(provider_file.parent.relative_to(AIRFLOW_PROVIDERS_DIR).as_posix())
            path = Path(root, filename)
            if path.is_file() and path.name.endswith(".py"):
                ALL_PROVIDER_FILES.append(Path(root, filename))

def get_provider_id_from_relative_import_or_file(relative_path_or_file: str) -> str | None:
    provider_candidate = relative_path_or_file.replace(os.sep, ".").split(".")
    while provider_candidate:
        candidate_provider_id = ".".join(provider_candidate)
        if "google_vendor" in candidate_provider_id:
            candidate_provider_id = candidate_provider_id.replace("google_vendor", "google")
        if candidate_provider_id in ALL_PROVIDERS:
            return candidate_provider_id
        provider_candidate = provider_candidate[:-1]
    return None

def get_provider_id_from_import(import_name: str, file_path: Path) -> str | None:
    if not import_name.startswith(AIRFLOW_PROVIDERS_IMPORT_PREFIX):
        # skip silently - it's OK to get non-provider imports
        return None
    relative_provider_import = import_name[len(AIRFLOW_PROVIDERS_IMPORT_PREFIX) :]
    provider_id = get_provider_id_from_relative_import_or_file(relative_provider_import)
    if provider_id is None:
        relative_path_from_import = relative_provider_import.replace(".", os.sep)
        if relative_path_from_import.startswith(tuple(suspended_paths)):
            return None
        warnings.append(f"We could not determine provider id from import {import_name} in {file_path}")
    return provider_id

def get_imports_from_file(file_path: Path) -> list[str]:
    root = parse(file_path.read_text(), file_path.name)
    visitor = ImportFinder()
    visitor.visit(root)
    return visitor.imports

def get_provider_id_from_file_name(file_path: Path) -> str | None:
    # is_relative_to is only available in Python 3.9 - we should simplify this check when we are Python 3.9+
    try:
        relative_path = file_path.relative_to(AIRFLOW_PROVIDERS_DIR)
    except ValueError:
        try:
            relative_path = file_path.relative_to(AIRFLOW_SYSTEM_TESTS_PROVIDERS_DIR)
        except ValueError:
            try:
                relative_path = file_path.relative_to(AIRFLOW_TESTS_PROVIDERS_DIR)
            except ValueError:
                errors.append(f"Wrong file not in the providers package = {file_path}")
                return None
    provider_id = get_provider_id_from_relative_import_or_file(str(relative_path))
    if provider_id is None and file_path.name not in ["__init__.py", "get_provider_info.py"]:
        if relative_path.as_posix().startswith(tuple(suspended_paths)):
            return None
        else:
            warnings.append(f"We had a problem to classify the file {file_path} to a provider")
    return provider_id

def check_if_different_provider_used(file_path: Path) -> None:
    file_provider = get_provider_id_from_file_name(file_path)
    if not file_provider:
        return
    imports = get_imports_from_file(file_path)
    for import_name in imports:
        imported_provider = get_provider_id_from_import(import_name, file_path)
        if imported_provider is not None and imported_provider not in ALL_PROVIDERS:
            warnings.append(f"The provider {imported_provider} from {file_path} cannot be found.")
        elif imported_provider and file_provider != imported_provider:
            ALL_DEPENDENCIES[file_provider]["cross-providers-deps"].append(imported_provider)

if __name__ == "__main__":
    find_all_providers_and_provider_files()
    num_files = len(ALL_PROVIDER_FILES)
    num_providers = len(ALL_PROVIDERS)
    console.print(f"Found {len(ALL_PROVIDERS)} providers with {len(ALL_PROVIDER_FILES)} Python files.")
    for file in ALL_PROVIDER_FILES:
        check_if_different_provider_used(file)
    for provider, provider_yaml_content in ALL_PROVIDERS.items():
        if not provider_yaml_content.get("suspended"):
            ALL_DEPENDENCIES[provider]["deps"].extend(provider_yaml_content["dependencies"])
    if warnings:
        console.print("[yellow]Warnings!\n")
        for warning in warnings:
            console.print(f"[yellow] {warning}")
        console.print(f"[bright_blue]Total: {len(warnings)} warnings.")
    if errors:
        console.print("[red]Errors!\n")
        for error in errors:
            console.print(f"[red] {error}")
        console.print(f"[bright_blue]Total: {len(errors)} errors.")
    unique_sorted_dependencies: dict[str, dict[str, list[str]]] = defaultdict(dict)
    for key in sorted(ALL_DEPENDENCIES.keys()):
        unique_sorted_dependencies[key]["deps"] = sorted(ALL_DEPENDENCIES[key]["deps"])
        unique_sorted_dependencies[key]["cross-providers-deps"] = sorted(
            set(ALL_DEPENDENCIES[key]["cross-providers-deps"])
        )
        excluded_versions = ALL_PROVIDERS[key].get("excluded-python-versions")
        unique_sorted_dependencies[key]["excluded-python-versions"] = (
            excluded_versions if excluded_versions else []
        )
    if errors:
        console.print()
        console.print("[red]Errors found during verification. Exiting!")
        console.print()
        sys.exit(1)
    old_dependencies = DEPENDENCIES_JSON_FILE_PATH.read_text()
    new_dependencies = json.dumps(unique_sorted_dependencies, indent=2) + "\n"
    if new_dependencies != old_dependencies:
        DEPENDENCIES_JSON_FILE_PATH.write_text(json.dumps(unique_sorted_dependencies, indent=2) + "\n")
        if os.environ.get("CI"):
            console.print()
            console.print(f"[info]Written {DEPENDENCIES_JSON_FILE_PATH}")
            console.print(
                f"[yellow]You will need to run breeze locally and commit "
                f"{DEPENDENCIES_JSON_FILE_PATH.relative_to(AIRFLOW_SOURCES_ROOT)}!\n"
            )
            console.print()
        else:
            console.print()
            console.print(
                f"[yellow]Regenerated new dependencies. Please commit "
                f"{DEPENDENCIES_JSON_FILE_PATH.relative_to(AIRFLOW_SOURCES_ROOT)}!\n"
            )
            console.print(f"[info]Written {DEPENDENCIES_JSON_FILE_PATH}")
            console.print()
    else:
        console.print(
            "[green]No need to regenerate dependencies!\n[/]"
            f"The {DEPENDENCIES_JSON_FILE_PATH.relative_to(AIRFLOW_SOURCES_ROOT)} is up to date!\n"
        )


import string
import json

from urllib.parse import urlencode

from django import template

from CMS.enums import enums
from CMS.translations import DICT
from courses.models import STUDENT_SATISFACTION_KEY, ENTRY_INFO_KEY, AFTER_ONE_YEAR_KEY, AFTER_COURSE_KEY, \
    ACCREDITATION_KEY

register = template.Library()

SHOW_STATS_LOOKUP = {
    STUDENT_SATISFACTION_KEY: 'show_satisfaction_stats',
    ACCREDITATION_KEY: 'accreditations',
    ENTRY_INFO_KEY: 'show_entry_information_stats',
    AFTER_ONE_YEAR_KEY: 'show_after_one_year_stats',
    AFTER_COURSE_KEY: 'show_after_course_stats'
}

@register.simple_tag
def queryparams(*_, **kwargs):
    safe_args = {key: value for key, value in kwargs.items() if value is not None}
    if safe_args:
        return '?{}'.format(urlencode(safe_args))
    return ''

@register.simple_tag
def get_translation(*_, **kwargs):
    key = kwargs.get('key')
    language = kwargs.get('language')
    string = DICT.get(key).get(language) if key in DICT else key

    if not string:
        string = DICT.get(key).get(enums.languages.ENGLISH) if key in DICT else ''

    if 'substitutions' in kwargs:
        string = string % kwargs.get('substitutions')
    return string

@register.simple_tag
def create_list(*args):
    return args

@register.simple_tag
def insert_values_to_rich_text(*_, **kwargs):
    list(kwargs.get('substitutions'))
    return kwargs.get('content').source % (kwargs.get('substitutions'))

@register.simple_tag
def length_of_list(view_list):
    return len(view_list)

@register.simple_tag
def map_distance_learning_values(key, language):
    if key in DICT.get('distance_learning_values'):
        return DICT.get('distance_learning_values').get(key).get(language)
    return key

@register.simple_tag
def should_show_accordion(courses, accordion_type):
    if accordion_type == ACCREDITATION_KEY:
        if type(courses) == tuple:
            show = False
            for course in courses:
                if course and course.accreditations:
                    show = True
            return show
        return courses.accreditations
    return True

@register.simple_tag
def title_to_id(title):
    return title.replace(' ', '_').lower()

@register.simple_tag
def get_alphabet():
    return list(string.ascii_lowercase)

@register.simple_tag
def get_max_length(list1, list2):
    return max(len(list1), len(list2))

@register.filter(name='times')
def times(number):
    return range(number)

@register.simple_tag
def get_index(index, view_list):
    if len(view_list) > index:
        return view_list[index]
    return None

@register.simple_tag
def get_course_name(course, is_english):
    name = ''
    if course.get('qualification'):
        name += course.get('qualification') + ' '
    if course.get('honours_award') and course.get('honours_award') == 1:
        name += '(Hons) '
    title = course.get('title')
    if title and is_english:
        name += title.get('english') if title.get('english') else title.get('welsh')
    else:
        name += title.get('welsh') if title.get('welsh') else title.get('english')
    return name

@register.simple_tag
def get_course_locations_list(locations, is_english):
    locations_list = []
    if is_english:
        for location in locations:
            location_name = location.get('english') if location.get('english') else location.get('welsh')
            locations_list.append(location_name)
    else:
        for location in locations:
            location_name = location.get('welsh') if location.get('welsh') else location.get('english')
            locations_list.append(location_name)
    return ','.join(locations_list)

@register.simple_tag
def is_multiple_of(number, base):
    return number % base == 0


# preprocess_vcf.py
# used to filter and create a VCF with per-genotype samples

from cyvcf2 import VCF, Writer
import pandas as pd
from snphwe import snphwe
import argparse

def build_args():
    parser = argparse.ArgumentParser(description='Filter vcf by \% called and HWE')
    parser.add_argument('vcffile', help='VCF to filter')
    parser.add_argument('--outfile', required=False, default='', help='Output for filtered VCF')
    parser.add_argument('--hwefile', required=False, default='', help='Output for HWE-rejected VCF')
    parser.add_argument('--uncalled', default=0.4, type=float, help='Filter out variant with > this fraction of uncalled genotypes (default 0.4)')
    parser.add_argument('--hwe', default=0.0001, type=float, help='HWE p value to reject (default 0.0001)')
    parser.add_argument('--maf', default=10, type=int, help='Require at least this # of het or homAlt genotypes (default 10)')
    parser.add_argument('--ref', default=10, type=int, help='Require at least this # of homRef genotypes (default 10)')
    return parser.parse_args()

def make_vcfs(vcf_filenames, gt_file): # input a list of VCF filenames and a per-sample genotypes file
    genotypes = pd.read_csv(gt_file, sep='\t', index_col=0)
    filenames = vcf_filenames
    for filename in filenames:
        old_vcf = pd.read_csv(filename, sep='\t', comment='#', names=['CHROM', 'POS', 'ID', 'REF', 'ALT', 'QUAL', 'FILTER', 'INFO', 'FORMAT', 'N/A'])
        old_vcf = old_vcf.drop('N/A', axis='columns')
        old_vcf['FORMAT'] = 'GT'

        geno_interest = genotypes.loc[old_vcf['ID']].set_index(old_vcf.index)
        new_vcf = pd.concat([old_vcf, geno_interest], axis=1)
        new_vcf.to_csv(filename[:-4]+'.per_sample.vcf', index=False, sep='\t')
        # I manually pasted in the metadata from the VCFs

# this requires per-sample VCFs
def filter_vcfs(args):
    vcf_reader = VCF(args.vcffile, gts012=True) # sets gt: 0 = homRef, 1 = het, 2 = homAlt, 3 = unknown
    nsamples = len(vcf_reader.samples)
    if (args.outfile != ''):
        vcf_writer = Writer(args.outfile, vcf_reader)
    if (args.hwefile != ''):
        hwe_writer = Writer(args.hwefile, vcf_reader)
    overall_count = 0
    uncalled_count = 0
    hwe_count = 0
    maf_count = 0
    for v in vcf_reader:
        overall_count += 1

        gts = v.gt_types
        ref = len(gts[gts == 0])
        het = len(gts[gts == 1])
        alt = len(gts[gts == 2])
        unc = len(gts[gts == 3])

        # check number uncalled
        if ((unc / nsamples) > args.uncalled): # too many uncalled
            uncalled_count += 1
            continue

        # check HW equilibrium
        called_N = het + alt + ref
        n_het = round(het * called_N / nsamples)
        n_alt = round(alt * called_N / nsamples)
        n_ref = called_N - n_het - n_alt
        p = snphwe(n_het, n_alt, n_ref)
        if (p < args.hwe): # does not follow HWE expectation
            hwe_count += 1
            if (args.hwefile != ''):
                hwe_writer.write_record(v)
            continue

        # check number alt and ref
        if ((het + alt) < args.maf or ref < args.ref):
            maf_count += 1
            continue

        if (args.outfile != ''):
            hwe_writer.write_record(v)

    if (args.outfile != ''):
        vcf_writer.close()
    if (args.hwefile != ''):
        hwe_writer.close()
    vcf_reader.close()

    print('Saw %d variants overall' % overall_count)
    print('Filtered out %d due to too many uncalled' % uncalled_count)
    print('Filtered out %d due to breaking HWE assumption' % hwe_count)
    print('Filtered out %d due to not enough Alt or Ref genotpyes' % maf_count)
    print('Left with %d variants' % (overall_count - uncalled_count - hwe_count - maf_count))

if __name__ == '__main__':
    args = build_args()
    filter_vcfs(args)


import logging
# logging.basicConfig(level=logging.INFO, datefmt='%Y/%m/%d %H:%M',format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# 创建日志器,每一个日志器都是唯一的
logger1 = logging.getLogger(__name__) # __name__代表的是__main__
logger2 = logging.getLogger('logger2')
logger3 = logging.getLogger('logger3')

# 创建记录
logger1.info('This is a log info',exc_info=True)
logger2.warning('Warning exists',)
logger3.error('%s - error %s ',stack_info=True,extra={'user':'tom','ip':'47.98.53.222'})


# -*- coding: utf-8 -*-
##----------------------------------------------------------------------
## Service model
##----------------------------------------------------------------------
## Copyright (C) 2007-2015 The NOC Project
## See LICENSE for details
##----------------------------------------------------------------------

# Third-party modules
from peewee import CharField, IntegerField, ForeignKeyField
from playhouse.signals import Model, post_save
# Tower modules
from db import db
from environment import Environment
from pool import Pool
from node import Node

class Service(Model):
    class Meta:
        database = db
        db_table = "service"

    environment = ForeignKeyField(Environment, on_delete="RESTRICT")
    service = CharField()
    pool = ForeignKeyField(Pool, null=True)
    node = ForeignKeyField(Node)
    n_instances = IntegerField(default=0)
    loglevel = CharField(default="info", choices=[
        "notset",
        "debug",
        "info",
        "warning",
        "error",
        "critical"
    ])


import sentencepiece as spm
import collections

SPIECE_UNDERLINE = "▁"

def load_vocab(vocab_file):
    """Loads a vocabulary file into a dictionary."""
    vocab = collections.OrderedDict()
    with open(vocab_file, "r", encoding="utf-8") as reader:
        tokens = reader.readlines()
    for index, token in enumerate(tokens):
        token = token.rstrip("\n")
        vocab[token] = index
    return vocab

class XLMRTokenizer(object):
    def __init__(self, vocab_file,special_token=''):
        sp_model = spm.SentencePieceProcessor()
        sp_model.Load(str(vocab_file))
        self.sp_model = sp_model
        self.bos_token = "<s>"
        self.eos_token = "</s>"
        self.sep_token = "</s>"
        self.cls_token = "<s>"
        self.unk_token = "<unk>"
        self.pad_token = "<pad>"
        self.mask_token = "<mask>"

        self.fairseq_tokens_to_ids = {"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

        # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
        self.fairseq_offset = 1

        self.fairseq_tokens_to_ids["<mask>"] = len(sp_model) + self.fairseq_offset
        self.fairseq_ids_to_tokens = {v: k for k, v in self.fairseq_tokens_to_ids.items()}

        self.cls_token_id = self._cls_token_id()
        self.sep_token_id = self._sep_token_id()
        self.pad_token_id = self._pad_token_id()
        self.mask_token_id = self._mask_token_id()
        self.eos_token_id = self._eos_token_id()

    def _cls_token_id(self):
        """ Id of the classification token in the vocabulary. E.g. to extract a summary of an input sequence leveraging self-attention along the full depth of the model. Log an error if used while not having been set. """
        return self._convert_token_to_id(self.cls_token)
    def _sep_token_id(self):
        """ Id of the classification token in the vocabulary. E.g. to extract a summary of an input sequence leveraging self-attention along the full depth of the model. Log an error if used while not having been set. """
        return self._convert_token_to_id(self.sep_token)
    def _pad_token_id(self):
        return self._convert_token_to_id(self.pad_token)
    def _eos_token_id(self):
        return self._convert_token_to_id(self.eos_token)
    def _mask_token_id(self):
        return self._convert_token_to_id(self.mask_token)

    def build_inputs_with_special_tokens(
            self,token_ids_0, token_ids_1):
        """
        Build model inputs from a sequence or a pair of sequence for sequence classification tasks
        by concatenating and adding special tokens.
        A BERT sequence has the following format:
        - single sequence: ``[CLS] X [SEP]``
        - pair of sequences: ``[CLS] A [SEP] B [SEP]``
        Args:
            token_ids_0 (:obj:`List[int]`):
                List of IDs to which the special tokens will be added
            token_ids_1 (:obj:`List[int]`, `optional`, defaults to :obj:`None`):
                Optional second list of IDs for sequence pairs.
        Returns:
            :obj:`List[int]`: list of `input IDs <../glossary.html#input-ids>`__ with the appropriate special tokens.
        """
        if token_ids_1 is None:
            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
        cls = [self.cls_token_id]
        sep = [self.sep_token_id]
        return cls + token_ids_0 + sep + token_ids_1 + sep

    @property
    def vocab_size(self):
        return len(self.sp_model) + self.fairseq_offset + 1  # Add the <mask> token

    def get_vocab(self):
        vocab = {self._convert_id_to_token(i): i for i in range(self.vocab_size)}
        return vocab

    def _tokenize(self,text):
        return self.sp_model.EncodeAsPieces(text)

    def _convert_token_to_id(self,token):
        """ Converts a token (str) in an id using the vocab. """
        if token in self.fairseq_tokens_to_ids:
            return self.fairseq_tokens_to_ids[token]
        spm_id = self.sp_model.PieceToId(token)

        # Need to return unknown token if the SP model returned 0
        return spm_id + self.fairseq_offset if spm_id else self.fairseq_tokens_to_ids[self.unk_token]

    def encode(self,text,text_b=None):
        tokens = self._tokenize(text)
        input_ids = []
        for token in tokens:
            input_ids.append(self._convert_token_to_id(token))

        input_ids_b = None
        if text_b is not None:
            input_ids_b = []
            tokens_b = self._tokenize(text_b)
            for token in tokens_b:
                input_ids_b.append(self._convert_token_to_id(token))

        #input_ids = self.build_inputs_with_special_tokens(input_ids,input_ids_b)

        return input_ids

    def decode(self,token_ids):
        _out = []
        for token_id in token_ids:
            _out.append(self._convert_id_to_token(token_id))
        return self.convert_tokens_to_string(_out)

    def _convert_id_to_token(self,index):
        """Converts an index (integer) in a token (str) using the vocab."""
        if index in self.fairseq_ids_to_tokens:
            return self.fairseq_ids_to_tokens[index]
        return self.sp_model.IdToPiece(index - self.fairseq_offset)
    def convert_tokens_to_string(self, tokens):
        """Converts a sequence of tokens (strings for sub-words) in a single string."""
        out_string = "".join(tokens).replace(SPIECE_UNDERLINE, " ").strip()
        return out_string


# -*- coding: utf-8 -*-
"""
Created on Fri Oct 18 14:35:09 2019

@author: sdorle
"""

# # Importing Libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

# # Importing Dataset
dataset = pd.read_csv("data\\Train_data_v4.csv")

# Correlation matrix
corr_matrix = dataset.corr()

# # Temp dataframe for storing only required independent variables
temp = dataset[['DOW', 'segment_duration_sec', 'Break_mins_per_hour', 'Reach', 'Time_hour', 'End_hour']]#, 'start_minutes', 'end_minutes']]#, 'Total_Duration', 'start_sec']]#]]
                #'start_sec', 'End_time', 'start_minutes', 'end_minutes']]#'AD_DURATION', 'PROMO_DURATION', 'segment_no']]#'Start_time_min']]#,'Reach']]
y = dataset['Break_min_filter'] # Dependent Variable
#temp = temp.head(100)
#y = y.head(100)

X_train = temp.values
y_train = y.values

#from sklearn.model_selection import train_test_split

#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2)

#################### Model Training #######################
from sklearn.ensemble import RandomForestClassifier
# Fitting Random Forest Classification to the Training set
classifier = RandomForestClassifier(n_estimators = 10, criterion = 'entropy', random_state = 42)
classifier.fit(X_train, y_train)

###### Test File #########
test = pd.read_csv("data\\\\test\\Test_data_v6.csv")#Test_data_v6.csv")#
temp = test[['DOW', 'tape_duration_sec', 'Break_mins_per_hour', 'Reach', 'Time_hour', 'End_hour']]#, 'start_min', 'end_min']]#, 'Total_Duration', 'start_sec']]# 'End_time',
             #'start_minutes', 'end_minutes']]#,'AD_DURATION',
             #'PROMO_DURATION', 'segment_no']]
X_test = temp.values
y_test = test['Break_min_filter']
y_test = y_test.values

#### Prediction #####
y_pred = classifier.predict(X_test)

##### Accuracy #####
from sklearn.metrics import accuracy_score
accuracy_score(y_test, y_pred)

new = pd.DataFrame()
new['act'] = y_test
new['pred'] = y_pred

# Making the Confusion Matrix
print(pd.crosstab(y_test, y_pred, rownames=['Actual'], colnames=['Predicted']))

# =============================================================================
# SVM
# =============================================================================
# training a linear SVM classifier
from sklearn.svm import SVC
svm_model_linear = SVC(kernel = 'linear', C = 1).fit(X_train, y_train)
svm_predictions = svm_model_linear.predict(X_test)

# model accuracy for X_test
accuracy = svm_model_linear.score(X_test, y_test)

# creating a confusion matrix
cm = confusion_matrix(y_test, svm_predictions)

from sklearn import svm
SVM = svm.SVC(decision_function_shape="ovo").fit(X_train, y_train)
SVM.predict(X_test)
round(SVM.score(X_test, y_test), 4)

# =============================================================================
# Performance Evaluation
# =============================================================================

from sklearn.metrics import r2_score
print('R Squared:', r2_score(y_test, y_pred))

#############################################
from sklearn.naive_bayes import MultinomialNB
gnb = MultinomialNB(alpha = 0.01)
model = gnb.fit(X_train, y_train)
#### Prediction #####
y_pred2 = model.predict(X_test)


# -*- coding: utf-8 -*-
"""
Created on Mon Apr 22 14:43:30 2019

@author: Mohammadreza
"""

import cv2
import numpy as np
import math
from matplotlib import pyplot as plt
import csv

#read image
img = cv2.imread('Dataset.jpg')

#grayscale
gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
cv2.waitKey(0)

#binarize
ret,thresh = cv2.threshold(gray,127,1,cv2.THRESH_BINARY_INV)
cv2.waitKey(0)
#cv2.THRESH_BINARY_INV
#find contours

dimensions = gray.shape
height = gray.shape[0]
width = gray.shape[1]

#print(height)

im2,ctrs, hier = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)

#sort contours
sorted_ctrs = sorted(ctrs, key=lambda ctr: cv2.boundingRect(ctr)[0])

for i, ctr in enumerate(sorted_ctrs):
    # Get bounding box
    x, y, w, h = cv2.boundingRect(ctr)

    # Getting ROI
    roi = gray[y:y+h, x:x+w]

    cv2.waitKey(0)
    new_height1 = 64
    new_width1 = 64
    new_image = cv2.resize(roi ,(new_height1,new_width1))
    cv2.imwrite(str(i) + '.jpg', new_image)
    cv2.rectangle(gray,(x,y),( x + w, y + h ),(90,0,255),2)
    ret1,thresh1 = cv2.threshold(new_image,127,1,1)

    new_image = thresh1

    #cv2.imshow('window', new_image)

    A = np.zeros((new_height1,new_width1))
    n = int(math.log(new_width1,2))
   # print(n)
    for e in range(new_width1):
        for j in range(new_height1):
            P = 1
            P_bin = bin(e)[2:].zfill(n)
            P_bin1 = bin(j)[2:].zfill(n)
            for v in range(n):
                D = int(P_bin[v*-1-1]) * int(P_bin1[v])
                E = int(np.power(-1,D))
                P = P * E
            A[e][j] = P
    Walsh_Image = A * new_image * A

    l = np.array(Walsh_Image)
    print(l)
    #l1 = l[:,1]
    with open('DataSet.csv', 'a', newline="") as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(Walsh_Image.flatten())
    print(l[:,1])
    #plt.imshow(Walsh_Image,cmap = 'gray')
    #plt.show()


#！ /usr/bin/env python3
# -*- coding:utf-8 -*-

import pymysql

db = pymysql.connect('192.168.43.254', 'root', '990830', db='mysql')
cur = db.cursor()
cur.execute("select User, host from user")
for r in cur:
    print(r)

cur.close()
db.close()


#!/usr/bin/env python3
from collections import defaultdict
import sys
sys.setrecursionlimit(10**6)

n = int(input())
c = list(map(int, input().split()))

data = defaultdict(list)
for _ in range(n-1):
    a, b = list(map(int, input().split()))
    data[a-1].append(b-1)
    data[b-1].append(a-1)

ans = []
visited = set()
used_color = defaultdict(int)

ans.append(1)
visited.add(0)
used_color[c[0]] += 1

# print(data)

def dfs(now_ver):
    # print(now_ver, visited)
    for next_ver in data[now_ver]:
        if next_ver in visited:
            continue
        else:
            if used_color[c[next_ver]] == 0:
                ans.append(next_ver+1)
            else:
                pass
            visited.add(next_ver)
            used_color[c[next_ver]] += 1
            dfs(next_ver)
            visited.remove(next_ver)
            used_color[c[next_ver]] -= 1

dfs(0)

ans.sort()

for i in ans:
    print(i)


"""
Implementation of multiple visualization algorithms.
Uses the following repos:

    - Repo: nn_interpretability
        - Autor: hans66hsu
        - URL: https://github.com/hans66hsu/nn_interpretability

    - Repo:
        - Author: Yuchi Ishikawa
        - URL: https://github.com/yiskw713/SmoothGradCAMplusplus
"""

import argparse
import os
import sys
import warnings
from itertools import combinations
from pathlib import Path

CURRENT = os.path.dirname(os.path.abspath(__file__))
sys.path.append(str(Path(CURRENT).joinpath("pytorch_cnn_visualizations", "src")))

import numpy as np
import torch
from matplotlib import pyplot as plt
from model import LitModel
from nn_interpretability.interpretation.am.general_am import ActivationMaximization
from nn_interpretability.interpretation.saliency_map.saliency_map import SaliencyMap
from PIL import Image
from skimage.color import rgb2gray
from skimage.io import imsave
from SmoothGradCAMplusplus.cam import GradCAM, SmoothGradCAMpp
from SmoothGradCAMplusplus.utils.visualize import reverse_normalize, visualize
from torch.nn import functional as F
from torchvision import transforms
from torchvision.utils import save_image
from tqdm import tqdm

def parse_args(vis_list):
    """
    Parses args

    Parameters
    ----------
    vis_list : str
        Possible visualizations

    Returns
    -------
    argsparse.Namespace
        Namespace with all args
    """
    # Handle args parsing
    parser = argparse.ArgumentParser(description="Calls visualization algorithms")
    parser.add_argument(
        "--dest",
        dest="dest",
        metavar="Destination",
        required=True,
        type=str,
        help="Destination path, subfolder structure will be restored",
    )
    parser.add_argument(
        "--filetype",
        dest="filetype",
        metavar="Filetype",
        type=str,
        default="ppm",
        help="File type of the images inside the subfolders (without '.'). By default 'ppm'",
    )
    parser.add_argument(
        "--model",
        metavar="Model",
        dest="model",
        type=str,
        required=True,
        help="Path to model file (.pt)",
    )
    parser.add_argument(
        "--num",
        metavar="Number Images",
        dest="num_images",
        type=int,
        default=50,
        help="How many images per folder should be (randomly) selected (default: 50)",
    )
    parser.add_argument(
        "--src",
        dest="src",
        metavar="Source",
        type=str,
        help="Path to source directory, script runs over subfolders (default: current Directory)",
    )
    parser.add_argument(
        "--vis",
        metavar="Visualization",
        dest="vis",
        type=str,
        help="Visualization algorithems that should be used (default: all). Choose from: "
        + str(vis_list),
    )
    parser.add_argument("--mean", dest="mean", nargs="+", required=True)

    parser.add_argument("--std", dest="std", default=None, nargs="+", required=True)

    args = parser.parse_args()

    # Set source
    if args.src is None:
        Path(os.getcwd())
    else:
        Path(args.src)

    # Set vis
    args.vis = args.vis.split("/")

    # Cast mean/std to floag
    args.mean = [float(i) for i in args.mean]
    args.std = [float(i) for i in args.std]

    return args

def get_algo_combinations(algos: list) -> str:
    """
    Creates String with all possible combinations of the implemented visualization algorithms

    Parameters
    ----------
    algos : list
        List with all visualization algorithms

    Returns
    -------
    str
        All possible combinations
    """
    subsets = []
    for L in range(0, len(algos) + 1):
        for subset in combinations(algos, L):
            subsets.append(subset)

    subsets_str = []
    for combination in subsets:
        if len(combination) > 0:
            combination_str = ""
            for el in combination:
                combination_str += el if len(combination_str) == 0 else "/" + el
            subsets_str.append(combination_str)

    return subsets_str

def prep_image(org_image: str, mean: list, std: list) -> torch.Tensor:
    """
    Prepares image. This includes the normalization and the transformation into a tensor

    Parameters
    ----------
    org_image : str
        Path to original image
    mean : list
        List with mean values for the rgb channels
    std : list
        List with std values for rgb channels

    Returns
    -------
    tensor
        Prepared image tensor
    """
    ex_image = Image.open(org_image)
    normalize = transforms.Normalize(mean, std)
    preprocess = transforms.Compose([transforms.ToTensor(), normalize])

    tensor = preprocess(ex_image)
    prep_img = tensor.unsqueeze(0)
    return prep_img

def activation_maximation(
    model: LitModel,
    target_class: str,
    org_image: torch.Tensor,
    dest: str,
    mean: list,
    std: list,
    device: str,
) -> None:
    """
    Performs activation maximation for the passed class

    Parameters
    ----------
    model : LitModel
        Model for activation maximation
    target_class : str
        Target class
    org_image : torch.Tensor
        Mean image of dataset
    dest : str
        Path to destination folder
    mean : list
        Mean values for all three channels of the train dataset
    std : list
        Mean values for all three channels of the train dataset
    device : str
        Device that should be used ('cpu'/'cuda')
    """
    # Params
    img_shape = (3, 48, 48)
    lr = 0.001
    reg_term = 1e-2
    epochs = 1000
    threshold = 0.995

    # Random image as start
    start_img = torch.rand((1, 3, 48, 48)).to(device)
    mean_img = org_image  # Mean was calculated in mean and passed as org_image

    # Preprocess
    normalize = transforms.Normalize(mean, std)
    transforms.Compose([transforms.ToTensor(), normalize])

    # Get Interpreter
    interpretor = ActivationMaximization(
        model=model,
        classes=[],
        preprocess=None,
        input_size=img_shape,
        start_img=start_img,
        class_num=int(target_class),
        lr=lr,
        reg_term=reg_term,
        class_mean=mean_img,
        epochs=epochs,
        threshold=threshold,
    )

    end_point = interpretor.interpret()

    # Calc score
    scores = model(end_point.to(device)).to(device)
    prob = torch.nn.functional.softmax(scores, 1)[0][int(target_class)] * 100
    print(f"Class {target_class}: {prob}")

    # Restore Image (unnormalize)
    x_restore = end_point.reshape(img_shape) * torch.tensor(std).view(
        3, 1, 1
    ) + torch.tensor(mean).view(3, 1, 1)
    image_restored = x_restore.permute(1, 2, 0)

    # Save Image
    image_dest = Path(dest).joinpath("activation_maximation", target_class)
    image_dest.parent.mkdir(exist_ok=True, parents=True)

    plt.imsave(f"{image_dest}.ppm", image_restored.numpy())

def salience_map(
    model: LitModel,
    _,
    org_image: torch.tensor,
    dest: str,
    mean: list,
    std: list,
    device: str,
) -> None:
    """
    Performs saliency map

    Parameters
    ----------
    model : LitModel
        Model for activation maximation
    target_class : _
        Not needed
    org_image : torch.Tensor
        Mean image of dataset
    dest : str
        Path to destination folder
    mean : list
        Mean values for all three channels of the train dataset
    std : list
        Mean values for all three channels of the train dataset
    device : str
        Device that should be used ('cpu'/'cuda')
    """
    # Prep image
    prep_img = prep_image(org_image, mean, std)
    prep_img = prep_img.to(device)

    # Create SaliencyMap
    interpretor = SaliencyMap(model, [], [48, 48], None)

    # Creat Map
    endpoint = interpretor.interpret(prep_img)

    # Save map
    image_dest = Path(dest).joinpath(
        "heatmap_saliency", org_image.parents[0].name, org_image.name
    )
    image_dest.parents[0].mkdir(parents=True, exist_ok=True)

    heatmap = 255 * endpoint.squeeze()
    heatmap = heatmap.cpu()

    heatmap = rgb2gray(heatmap)
    grayscale_uint8 = heatmap.astype(np.uint8)
    imsave(str(image_dest), grayscale_uint8)

    # plt.imsave(str(image_dest), endpoint.cpu().squeeze(0), cmap="jet")

def grad_cam(
    model: LitModel,
    _,
    org_image: torch.tensor,
    dest: str,
    mean: list,
    std: list,
    device: str,
) -> None:
    """
    Performs GradCam

    Parameters
    ----------
    model : LitModel
        Model for activation maximation
    target_class : _
        Not needed
    org_image : torch.Tensor
        Mean image of dataset
    dest : str
        Path to destination folder
    mean : list
        Mean values for all three channels of the train dataset
    std : list
        Mean values for all three channels of the train dataset
    device : str
        Device that should be used ('cpu'/'cuda')
    """
    # Prep image
    prep_img = prep_image(org_image, mean, std)
    prep_img = prep_img.to(device)

    # Create SmoothGradCAM
    wrapped_model = GradCAM(model, model.features[13])

    cam, _ = wrapped_model(prep_img)
    img = reverse_normalize(prep_img)

    # Create heatmap
    _, _, H, W = img.shape
    cam = F.interpolate(cam, size=(H, W), mode="bilinear", align_corners=False)
    heatmap = 255 * cam.squeeze()
    heatmap = heatmap.cpu()

    # Save heatmap
    image_dest = Path(dest).joinpath(
        "heatmap_grad_cam", org_image.parents[0].name, org_image.name
    )
    image_dest.parents[0].mkdir(parents=True, exist_ok=True)
    heatmap = rgb2gray(heatmap)
    grayscale_uint8 = heatmap.astype(np.uint8)
    imsave(image_dest, grayscale_uint8)

    # Save image (overlay)
    heatmap = visualize(img.cpu(), cam.cpu())
    image_dest = Path(dest).joinpath(
        "grad_cam", org_image.parents[0].name, org_image.name
    )
    image_dest.parents[0].mkdir(parents=True, exist_ok=True)
    save_image(heatmap, str(image_dest))

def grad_cam_plus_plus(
    model: LitModel,
    _,
    org_image: torch.tensor,
    dest: str,
    mean: list,
    std: list,
    device: str,
) -> None:
    """
    Performs GradCam++

    Parameters
    ----------
    model : LitModel
        Model for activation maximation
    target_class : _
        Not needed
    org_image : torch.Tensor
        Mean image of dataset
    dest : str
        Path to destination folder
    mean : list
        Mean values for all three channels of the train dataset
    std : list
        Mean values for all three channels of the train dataset
    device : str
        Device that should be used ('cpu'/'cuda')
    """
    # Prep image
    prep_img = prep_image(org_image, mean, std)
    prep_img = prep_img.to(device)

    # Create SmoothGradCAMpp
    wrapped_model = SmoothGradCAMpp(
        model, model.features[13], n_samples=25, stdev_spread=0.15
    )

    cam, _ = wrapped_model(prep_img)
    img = reverse_normalize(prep_img)

    # Create heatmap
    _, _, H, W = img.shape
    cam = F.interpolate(cam, size=(H, W), mode="bilinear", align_corners=False)
    heatmap = 255 * cam.squeeze()
    heatmap = heatmap.cpu()

    # Save heatmap
    image_dest = Path(dest).joinpath(
        "heatmap_grad_cam_pp", org_image.parents[0].name, org_image.name
    )
    image_dest.parents[0].mkdir(parents=True, exist_ok=True)
    heatmap = rgb2gray(heatmap)
    grayscale_uint8 = heatmap.astype(np.uint8)
    imsave(image_dest, grayscale_uint8)

    # Save image (overlay)
    heatmap = visualize(img.cpu(), cam.cpu())
    image_dest = Path(dest).joinpath(
        "grad_cam_pp", org_image.parents[0].name, org_image.name
    )
    image_dest.parents[0].mkdir(parents=True, exist_ok=True)
    save_image(heatmap, str(image_dest))

def generate_mean_image(images: list, device: str) -> torch.tensor:
    """
    Generates mean images based on passed images

    Parameters
    ----------
    images : list
        List with image paths
    device : str
        'cpu' or 'cuda'

    Returns
    -------
    torch.tensor
        Mean image
    """
    image_sum = torch.zeros(images[0].size())

    for image in images:
        image_sum += image

    mean_image = (image_sum / len(images)).to(device)
    mean_image = (mean_image - mean_image.min()) / (mean_image.max() - mean_image.min())

    return mean_image.to(device)

def main(args, vis_to_function, device):
    """
    Calls visualization method

    Parameters
    ----------
    args : argparse.namespace
        Namespace with argumetns
    vis_to_function : str
        Target visualization method
    device : str
        'cpu' or 'cuda'
    """

    # Load model
    model = LitModel.load_from_checkpoint(
        args.model,
        mean=args.mean,
        std=args.std,
        train_dataset=None,
        test_dataset=args.src,
    )
    model.to(device)
    model.eval()

    # Loop over all subfolders (label)
    num_dir = len([x for x in Path(args.src).iterdir() if x.is_dir()])
    for idx, directory in tqdm(
        enumerate([x for x in Path(args.src).iterdir() if x.is_dir()]), total=num_dir
    ):

        # Get target class
        target_class = directory.name

        # Select random images
        files = [x for x in directory.glob(f"*.{args.filetype}") if x.is_file()]

        # Run passed algos over the selected images
        for algo in args.vis:
            if algo == "Activation Maximation":
                # Because AM acts on the complete class, we wont call the method on each image
                # Instead of that, we create the class mean and pass that into the method
                data_list = []
                for image in files:
                    # Generate Tensors
                    prep_img = prep_image(image, args.mean, args.std)
                    data_list.append(prep_img)

                class_mean_img = generate_mean_image(data_list, device).to(device)
                activation_maximation(
                    model,
                    directory.name,
                    class_mean_img,
                    args.dest,
                    args.mean,
                    args.std,
                    device,
                )

            else:
                for image in files:
                    globals()[vis_to_function.get(algo)](
                        model,
                        target_class,
                        image,
                        args.dest,
                        args.mean,
                        args.std,
                        device,
                    )

if __name__ == "__main__":
    warnings.filterwarnings("ignore")
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    possible_algos = ["Activation Maximation", "Saliency", "GradCam", "GradCam++"]
    vis_to_function = {
        "GradCam": "grad_cam",
        "GradCam++": "grad_cam_plus_plus",
        "Saliency": "salience_map",
        "Activation Maximation": "activation_maximation",
    }
    algo_combinations = get_algo_combinations(possible_algos)
    args = parse_args(algo_combinations)
    main(args, vis_to_function, device)


#!/usr/bin/env python

from __future__ import print_function
from prometheus_client import start_http_server
from prometheus_client.core import REGISTRY, Metric
import os
import sys
import time
import itertools
import operator
from lib import tiller
from collections import Counter
from hapi.release import status_pb2 as hapi_dot_release_dot_status__pb2

tiller_namespace = 'kube-system'

if 'ENV' in os.environ:
    if os.environ['ENV'] == 'dev':
        tiller_endpoint = '127.0.0.1'
else:
    if 'TILLER_NAMESPACE' in os.environ:
        tiller_namespace = os.environ['TILLER_NAMESPACE']
    tiller_endpoint = "tiller-deploy.%s" % tiller_namespace

class Release:
    def __init__(self, name, chart_name, namespace, status, version, app_version, revision):
        self.name = name
        self.chart_name = chart_name
        self.namespace = namespace
        self.status = status
        self.version = version
        self.app_version = app_version
        self.revision = revision

class CustomCollector(object):
    def __init__(self):
        max_retries = 5
        for i in range(max_retries):
            try:
                self.tiller = tiller.Tiller(host=tiller_endpoint)
            except Exception as e:
                print(e)
                continue
            else:
                break
        else:
            print("Failed to connect to tiller on %s" % tiller_endpoint)
            sys.exit(1)

    def get_unique(self, deployed_releases_raw, failed_releases_raw):
        unique = []
        unique_failed_releases = self.get_unique_releases(failed_releases_raw)
        deployed_releases = self.to_releases(deployed_releases_raw)
        for deployed_release in deployed_releases:
            isFailedRelease = False
            for failed_release in unique_failed_releases:
                if deployed_release.name == failed_release.name and failed_release.revision > deployed_release.revision:
                    isFailedRelease = True
                    unique.append(failed_release)
                    break

            if not isFailedRelease:
                unique.append(deployed_release)

        return unique

    def to_releases(self, releases_raw):
        releases = []
        for release_raw in releases_raw:
            release = Release(release_raw.name,
                                    release_raw.chart.metadata.name,
                                    release_raw.namespace,
                                    hapi_dot_release_dot_status__pb2._STATUS_CODE.values_by_number[release_raw.info.status.code].name,
                                    release_raw.chart.metadata.version,
                                    release_raw.chart.metadata.appVersion,
                                    release_raw.version)
            releases.append(release)
        return releases

    def get_unique_releases(self, raw_releases):
        unique = []
        releases = self.to_releases(raw_releases)
        get_attr = operator.attrgetter('name')

        new_list = [list(g) for k, g in itertools.groupby(sorted(releases, key=get_attr), get_attr)]

        for release_group in new_list:
            latest_release = max(release_group, key=operator.attrgetter('revision'))
            unique.append(latest_release)
        return unique

    def collect(self):
        while True:
            try:
                all_deployed_releases_raw = self.tiller.list_releases("DEPLOYED")
                all_failed_releases_raw = self.tiller.list_releases("FAILED")
                all_releases = self.get_unique(all_deployed_releases_raw, all_failed_releases_raw)
                break
            except Exception as e:
                print(e)
                continue
        metric = Metric('helm_chart_info', 'Helm chart information', 'gauge')
        chart_count = Counter([(release.name, release.chart_name, release.version, release.app_version, release.namespace, release.status, release.revision, tiller_namespace) for release in all_releases])
        for chart in chart_count:
            metric.add_sample(
                    'helm_chart_info',
                    value=chart_count[chart],
                    labels={"name": chart[0], "chart_name": chart[1], "version": chart[2], "app_version":  chart[3],"namespace": chart[4], "status": chart[5], "revision": str(chart[6]), "tiller_namespace": chart[7]}
             )
        yield metric

if __name__ == "__main__":
    start_http_server(9484)
    REGISTRY.register(CustomCollector())
    print('Serving metrics on http://localhost:9484')
    while True:
        time.sleep(30)


# -*- coding: utf-8 -*-
"""
Created on Fri Mar  6 14:25:48 2020

@author: Group B07
Worked on plotting
@version: 3.5 22/03/2020
"""

import numpy as np
import control.matlab as cm
import matplotlib.pyplot as plt
import numpy.linalg
from massbalance import *
from data_generator_complete import *
import warnings
import scipy.optimize as sp
import time
import sys
from progress.bar import Bar
#import sublime, sublime_plugin
#import time

warnings.filterwarnings("ignore")

plt.close('all')

# +++++++++++++++++++++++++++++++++ Helper Functions ++++++++++++++++++++++++++++++++++++++++++++++

# ---------------------------------- fitting ------------------------------

def double(t, A1, eps1, omega_d1, A2, eps2, omega_d2):
    return 1.2 + A1 * np.exp(eps1 * t) * np.sin(omega_d1 * t + 0) + A2 * np.exp(eps2 * t) * np.cos(omega_d2 * t + 0)

def doubleSP(t, A1, eps1, omega_d1, A2, eps2, omega_d2, phi01, phi02):
    return A1 * np.exp(eps1 * t) * np.sin(omega_d1 * t + phi01) + A2 * np.exp(eps2 * t) * np.cos(omega_d2 * t + phi02)

def simple(t, A1, eps1, omega_d1, phi01):
    return A1 * np.exp(eps1 * t) * np.sin(omega_d1 * t + phi01)

def short(t, A, eps, omega, B, C):
    return A * np.exp(t * eps) * np.cos(omega * t + b)

def calcT(eps):
    return -np.log(1 / 2) / abs(eps)

def calcP(omega):
    return 2 * np.pi / abs(omega)

# ----------------------------------- plots ---------------------------------
#def plotting(x, y, name, variable, unit, label_name="Simulation", title=None, mins=False):
    #"""Use this for plotting. It returns the figure so we can add more to it"""
    #ax = plt.figure(str(name))
    # ax.legend("best")

    ##if mins:
       # x /= 60  # change time to mins from secs
       # plt.xlabel("t [min]")
    #else:
        #plt.xlabel("t [s]")

    #if title != None:
        #plt.title(str(title))

    ##if label_name == "Simulation":
       # plt.plot(x - x[0], y, '--', label=label_name)
    #else:
       # plt.plot(x - x[0], y, label=label_name)

   # lab = str(str(variable) + " " + "[" + unit + "]")
   # plt.legend(loc='best')
   # plt.ylabel(lab)
   # plt.grid(True)
    # plt.savefig(title)
   # plt.show()

   # return ax

# +++++++++++++++++++++++++++++++++++ Global variables+++++++++++++++++++++++++++++++++++++++++++++++

# Citation 550 - Linear simulation
# Aircraft geometry

S = 30.00  # wing area [m^2]
Sh = 0.2 * S  # stabiliser area [m^2]
Sh_S = Sh / S  # [ ]
lh = 0.71 * 5.968  # tail length [m]
c = 2.0569  # mean aerodynamic cord [m]
lh_c = lh / c  # [ ]
b = 15.911  # wing span [m]
bh = 5.791  # stabilser span [m]
A = b ** 2 / S  # wing aspect ratio [ ]
Ah = bh ** 2 / Sh  # stabilser aspect ratio [ ]
Vh_V = 1  # [ ]
ih = -2 * np.pi / 180  # stabiliser angle of incidence [rad]

oew = 4157.174  # Operational Empty Weight [kg]
m_payload = 765  # Payload mass [kg]

# Aerodynamic properties
e = 0.5466  # Oswald factor [ ]
CD0 = 0.01980  # Zero lift drag coefficient [ ]
CLa = 4.547  # Slope of CL-alpha curve [ ]

# Constant values concerning atmosphere and gravity
rho0 = 1.2250  # air density at sea level [kg/m^3]
lam = -0.0065  # temperature gradient in ISA [K/m]
Temp0 = 288.15  # temperature at sea level in ISA [K]
R = 287.05  # specific gas constant [m^2/sec^2K]
g = 9.81  # [m/sec^2] (gravity constant)

# Loading in data

altlst = np.genfromtxt("Data_SI_correct/Dadc1_altSI.txt", skip_header=2)  # reading in the altitude values
taslst = np.genfromtxt("Data_SI_correct/Dadc1_tasSI.txt", skip_header=2)  # reading in the true airspeed values
aoalst = np.genfromtxt("Data_SI_correct/vane_AOASI.txt", skip_header=2)  # reading in the angle of attack
pitchlst = np.genfromtxt("Data_SI_correct/Ahrs1_bPitchRateSI.txt", skip_header=2)  # reading in the pitch
# Simulation parameters:
nsteps = 10 ** 3

# +++++++++++++++++++++++++++++++++++++++++ MAIN ++++++++++++++++++++++++++++++++++++++++++++++++++++
def main(t0, deltat, t, input_type, input_u, CZu, CXu, CZa, Cmq):
    """Input type: elevator
                    rudder
                    airleron"""

    # Find time
    idx = np.where(timelst == t0)[0]

    # Flight condition
    #  m_fuel    = 1197.484        # CHANGE Total fuel mass [kg]

    hp = altlst[idx]  # CHANGE pressure altitude in the stationary flight condition [m]
    V = taslst[idx]  # CHANGE true airspeed in the stationary flight condition [m/sec]
    alpha = np.radians(aoalst[idx])  # angle of attack in the stationary flight condition [rad]
    theta = np.radians(pitchlst[idx])  # pitch angle in the stationary flight condition [rad]
    gamma = theta - alpha  # CHANGE flight path angle -

    # Aircraft mass
    m = mass(t0)  # mass [kg]

    # Longitudinal stability
    Cma = -0.4435  # CHANGE longitudinal stabilty [ ]
    Cmde = -1.001  # CHANGE elevator effectiveness [ ]

    # air density [kg/m^3]
    rho = rho0 * pow(((1 + (lam * hp / Temp0))), (-((g / (lam * R)) + 1)))
    W = m * g  # [N]       (aircraft weight)

    # Aircraft inertia (depend on t0):
    muc = m / (rho * S * c)
    mub = m / (rho * S * b)
    KX2 = 0.019
    KZ2 = 0.042
    KXZ = 0.002
    KY2 = 1.25 * 1.114

    # Aerodynamic constants:

    Cmac = 0  # Moment coefficient about the aerodynamic centre [ ]
    CNwa = CLa  # Wing normal force slope [1/rad]
    CNha = 2 * np.pi * Ah / (Ah + 2)  # Stabiliser normal force slope [ ]
    depsda = 4 / (A + 2)  # Downwash gradient [ ]

    # Lift and drag coefficient (depend on t0):

    CL = 2 * W / (rho * V ** 2 * S)  # Lift coefficient [ ]
    CD = CD0 + (CLa * alpha) ** 2 / (np.pi * A * e)  # Drag coefficient [ ]

    # Stabiblity derivatives
    CX0 = W * np.sin(theta) / (0.5 * rho * V ** 2 * S)
    # CXu    = -0.095         #corrected
    CXa = +0.47966  # Positive! (has been erroneously negative since 1993)
    CXadot = +0.08330
    CXq = -0.28170
    CXde = -0.03728

    CZ0 = -W * np.cos(theta) / (0.5 * rho * V ** 2 * S)
    # CZu    = -0.37616
    CZa = -5.74340
    CZadot = -0.00350
    CZq = -5.66290
    CZde = -0.69612

    Cmu = +0.06990  # positive!
    Cmadot = +0.17800  # positive!
    Cmq = -8.79415

    #CYb = -0.7500
    #CYbdot = 0
    #CYp = -0.0304
    #CYr = +0.8495
    #CYda = -0.0400
    #CYdr = +0.2300

    #Clb = -0.10260
    #Clp = -0.71085
    #Clr = +0.23760
    #Clda = -0.23088
    #Cldr = +0.03440

    #Cnb = +0.1348
    #Cnbdot = 0
    #Cnp = -0.0602
    #Cnr = -0.2061
    #Cnda = -0.0120
    #Cndr = -0.0939

    # c-matrix dimensions
    s1 = (4, 4)
    s2 = (4, 1)
    s3 = (4, 2)

    # Creating the different c-matrices (c1, c2 &c3) for symmetrical flight
    # c1 matrix
    c1 = np.zeros(s1)
    c1[0, 0] = -2 * muc * (c / V)
    c1[1, 1] = (CZadot - 2 * muc) * (c / V)
    c1[2, 2] = -(c / V)
    c1[3, 1] = Cmadot * (c / V)
    c1[3, 3] = -2 * muc * KY2 * ((c / V) ** 2)

    # c2 matrix
    c2 = np.zeros(s1)
    c2[0, 0] = -CXu
    c2[0, 1] = -CXa
    c2[0, 2] = -CZ0
    c2[0, 3] = -CXq * (c / V)
    c2[1, 0] = -CZu
    c2[1, 1] = -CZa
    c2[1, 2] = -CX0
    c2[1, 3] = -(CZq + 2 * muc) * (c / V)
    c2[2, 3] = -(c / V)
    c2[3, 0] = -Cmu
    c2[3, 1] = -Cma
    c2[3, 3] = -Cmq * (c / V)

    # c3 matrix
    c3 = np.zeros(s2)
    c3[0, 0] = -CXde
    c3[1, 0] = -CZde
    c3[3, 0] = -Cmde

    # Creating the different c-matrices (c4, c5 &c6) for asymmetrical flight

    # Time responses for unit steps:
    # t = np.linspace(t0,t0+ deltat, nsteps) -t0
    u = input_u

    # print("u and t:",u,t,sep='\n')
    # print("u shape:",u.shape)
    # print("t shape:",t.shape)
    if t.shape != u.shape:
        print("Wrong slicing for input and time!\n")
        return -1

    # Now, we distinct between inputs:

    if input_type == "elevator":
        #print("Calculating for elevator input...")
        # Symmetric system is triggered:

        # Creating the state matrix(A) and the input matrix(B) for symmetrical flight - xdot = c1^-1*c2*x c1^-1*c3*u = Ax + Bu
        A_s = np.dot(np.linalg.inv(c1), c2)
        B_s = np.dot(np.linalg.inv(c1), c3)
        C_s = np.identity(4)
        D_s = np.zeros((4, 1))

        # System in state-space
        sys_s = cm.StateSpace(A_s, B_s, C_s, D_s)
        poles_s = cm.pole(sys_s)
        # print("Eigenvalues of the symmetric system: ", poles_s,sep='\n') #verified

        # Time responses for unit steps:
        yout, tout, uout = cm.lsim(sys_s, u, t)  # general time response

        u_out_s = yout[:, 0]
        alpha_out_s = yout[:, 1] + alpha
        theta_out_s = yout[:, 2] + theta
        q_out_s = yout[:, 3]

        # Plotting....
        # plotting(t,u_out_s,str("u Response for " +input_type+ " input, t0= "+ str(t0)),"u","m/s")
        # plotting(t,alpha_out_s,str("Alpha Response for " +input_type+ " input, t0= "+ str(t0)),r"$\alpha$","deg")
        # plotting(t,theta_out_s,str("Theta Response for " +input_type+ " input, t0= "+ str(t0)),r"$\theta$","deg")
        # plotting(t,q_out_s,str("q Response for " +input_type+ " input, t0= "+ str(t0)),"$q$",r"deg/s")
        # print("\tPlotted!")
        return theta_out_s, q_out_s, poles_s

    return 1

# ++++++++++++++++++++++++++++++++++++++ Input & Output +++++++++++++++++++++++++++++++++++++++++++++++++++

# Simulation parameters for dynamic measurements:
# input: ph -> elevator def
#       short period -> elevator def
#       dutch roll -> rudder def
#       dutch roll_yd -> rudder def
#       ar -> aileron def
#       spi -> rudder def (pulse-like input)

# output: ph -> pitch, pitch rate
#       shp -> pitch, pitch rate
#       dr -> yaw rate, roll rate
#       dr_yd -> yaw rate, roll rate
#       ar -> roll, roll_rate
#       spi -> roll, yaw_rate

#if __name__ == "__main__":

# print("Collecting data...")

# t0_lst         = [53.5*60,58.6*60+3,60.1*60+5,60.95*60+5,57.0*60,3746]           #s
# deltat_lst     = [148, 5, 28 ,19 ,60 ,50]                                 #s -- these should match data_generator.py values (at the end)
input_type_lst = ["elevator","elevator","rudder","rudder","aileron","aileron"]

################################# PHUGOID ###############################################
# print("Phugoid")
# t0, deltat, utime_ph, u_ph, u_ph_p, u_ph_p_rate = phugoid()
# fig_q = plotting(utime_ph ,u_ph_p_rate,str("q Response for " +input_type_lst[0]+ " input, t0= "+ str(t0)),"$q$",r"deg/s",label_name="Flight Test",title="Phugoid -> Pitch Rate")
# fig_theta = plotting(utime_ph ,u_ph_p,str("Theta Response for " +input_type_lst[0]+ " input, t0= "+ str(t0)),r"$\theta$",r"deg",label_name="Flight Test",title="Phugoid -> Pitch")
# eig_p = main(t0,deltat,utime_ph,input_type_lst[0],u_ph)
#
# #...debugging....working?!
# utime = utime_ph-utime_ph[0]                                            # translate the interval for better fitting
# coeffs,cov = sp.curve_fit(double,utime,u_ph_p, p0=[5,0.01,-0.12,0.1,-1,1.7])  #initial guess is IMPORTANT
# eig_test = np.sqrt(coeffs[1]**2 + coeffs[2]**2)                      #absolute value
# print("Eigenvalues phugoid model: ", eig_p[3])
# print("Eigenvalues phugoid period test: %r + j %r" %(coeffs[1],coeffs[2]))
#
##print(utime)
##print(coeffs,cov,sep="\n")
# plt.figure("Testing")
# plt.plot(utime,double(utime,*coeffs),'r')
# print("Phugoid relative error [%]: ", abs((abs(eig_p[3])-eig_test))*100/abs(eig_p[3]))   #first two are short period (large omega), last two are phugoid
# print("\nTest:T1/2 = %r [s] and P= %r [s]" %(round(calcT(coeffs[1]),3),round(calcP(coeffs[2]),3)))

######################################## SHORT PERIOD ############################################

# print("Shord period")
# t0, deltat, utime_shp, u_shp, u_shp_p, u_shp_p_rate = short_period()
# plotting(utime_shp,u_shp_p_rate,str("q Response for " +input_type_lst[1]+ " input, t0= "+ str(t0)),"$q$",r"deg/s",label_name="Flight Test", title="Short Period -> Pitch Rate")
# plotting(utime_shp,u_shp_p,str("Theta Response for " +input_type_lst[1]+ " input, t0= "+ str(t0)),r"$\theta$",r"deg",label_name="Flight Test", title="Short Period -> Pitch")
# eig_s = main(t0,deltat,utime_shp,input_type_lst[1],u_shp)
# # print(eig_s)
#
# # #...debugging....
# utime = utime_shp-utime_shp[0]                                            # translate the interval for better fitting
# coeffs,cov = sp.curve_fit(doubleSP,utime,u_shp_p_rate)  #initial guess is IMPORTANT
# eig_test = np.sqrt(coeffs[1]**2 + coeffs[2]**2)                      #absolute value
# print("Eigenvalues short period model: ", eig_s[1])
# print("Eigenvalues  short period test: %r + j %r" %(coeffs[1],coeffs[2]))
#
# # print(coeffs,cov,sep="\n")
# # plt.figure("Testing")
# # plt.plot(utime,doubleSP(utime,*coeffs),'r')
# # plt.plot(utime,u_shp_p_rate,'b')
#
# print("Eigenvalue error [%]: ", abs((abs(eig_s[1])-eig_test))*100/abs(eig_s[1]))   #first two are short period (large omega), last two are phugoid
# print("\nTest:T1/2 = %r [s] and P= %r [s]" %(round(calcT(coeffs[1]),3),round(calcP(coeffs[2]),3)))
#++++++++++++++++++++++++++++++++++++++ RMSE calculation +++++++++++++++++++++++++++++++++++++++++++++++++++

def RMSE(flight_data,simulation_data):
    return np.sqrt(((flight_data - simulation_data) ** 2).mean())

  #++++++++++++++++++++++++++++++++++++++ Phugoid Model Improvement +++++++++++++++++++++++++++++++++++++++++++++++++++
#RMSE_original_phugoid_p_rate = 0.18537027365097794
#RMSE_original_phugoid_p = 3.248328645062731

#t0, deltat, utime_ph, u_ph, u_ph_p, u_ph_p_rate = phugoid()
#utime = utime_ph-utime_ph[0]                                            # translate the interval for better fitting
#coeffs,cov = sp.curve_fit(double,utime,u_ph_p, p0=[5,0.01,-0.12,0.1,-1,1.7])  #initial guess is IMPORTANT
#eig_test = np.sqrt(coeffs[1]**2 + coeffs[2]**2)
#rel_error_final = 100
#CZu_final = -0.37616
#CXu_final = -0.095
#interval = 0.01
#step = 30
#lst_CXu =np.linspace(CXu_final - interval, CXu_final + interval, step)
#lst_CZu =np.linspace(CZu_final - interval, CZu_final + interval, step)
#from progress.bar import Bar
#bar = Bar('Processing', max=step**2)
#error_real_final = 100
#error_imag_final = 100
#min_RMSE_phugoid_p_rate = RMSE_original_phugoid_p_rate
#min_RMSE_phugoid_p = RMSE_original_phugoid_p
#for i in lst_CZu:
#  for j in lst_CXu:
#    bar.next()
#    theta_out_s_p, q_out_s_p, eig_p = main(t0,deltat,utime_ph,input_type_lst[0],u_ph, i, j, -5.74340, -0.00350, -5.66290, 0.17800, -8.79415) #i = CZu, j = CXu CZa, CZadot, CZq, Cmadot, Cmq
    # #...debugging....working?!
#    real_num = eig_p[3].real
#    imag_num = eig_p[3].imag
#    real_ft  = eig_test.real
#    imag_ft = eig_test.imag
#    error_real = abs(real_num - real_ft)
#    error_imag = abs(imag_num - real_ft)
#    rel_error = abs((abs(eig_p[3])-eig_test))*100/abs(eig_p[3])
#    if RMSE(q_out_s_p,u_ph_p_rate) < min_RMSE_phugoid_p_rate and  RMSE(theta_out_s_p,u_ph_p) < min_RMSE_phugoid_p:
#      if error_real < error_real_final and error_imag < error_imag_final:
#          print('x')
#          error_real_final = error_real
#          error_imag_final = error_imag
#          rel_error_final = rel_error
#          CZu_final = i
#          CXu_final = j
#   min_RMSE_phugoid_p_rate=RMSE(q_out_s_p,u_ph_p_rate)
#    min_RMSE_phugoid_p=RMSE(theta_out_s_p,u_ph_p)
#bar.finish()

#print("The relative error is:", rel_error_final)
#print("CZu = ", CZu_final)
#print("CXu = ", CXu_final)

#CZu =  -0.44616 -> period
#CXu =  -0.165 -> amplitude

#CXu_modified= -0.15 # best one so far -0.14500000000000002
#CZu_modified= -0.44616 # best one so far: -0.42616
########################################### Short Period Motion Model Improvement #############################
RMSE_original_short_period_p_rate = 100#0.39683309816093915
RMSE_original_short_period_p = 100#3.087676046949498
min_RMSE_short_period_p_rate = RMSE_original_short_period_p_rate
min_RMSE_short_period_p = RMSE_original_short_period_p
t0, deltat, utime_shp, u_shp, u_shp_p, u_shp_p_rate = short_period()
utime = utime_shp-utime_shp[0]                                            # translate the interval for better fitting
coeffs,cov = sp.curve_fit(doubleSP,utime,u_shp_p_rate)  #initial guess is IMPORTANT
eig_test = np.sqrt(coeffs[1]**2 + coeffs[2]**2)

rel_error_final = 100
CZa_final = -5.74340
#CZadot_final = -0.00350
#CZq_final = -5.66290
#Cmadot_final = 0.17800
Cmq_final = -8.79415
interval = 30
step = 30
lst_CZa =np.linspace(CZa_final - interval, CZa_final + interval, step)
#lst_CZadot =np.linspace(CZadot_final - interval, CZadot_final + interval, step)
#lst_CZq =np.linspace(CZq_final - interval, CZq_final + interval, step)
#lst_Cmadot =np.linspace(Cmadot_final - interval, Cmadot_final + interval, step)
lst_Cmq =np.linspace(Cmq_final - interval, Cmq_final + interval, step)
from progress.bar import Bar
bar = Bar('Processing', max=100)
error_real_final = 100
error_imag_final = 100
for i in lst_CZa:
#  for j in lst_Czadot:
    #for k in lst_CZq:
    #for l in lst_Cmadot:
  for j in lst_Cmq:
      bar.next()
      theta_out_s_shp, q_out_s_shp, eig_s = main(t0,deltat,utime_shp,input_type_lst[1],u_shp, -0.37616,-0.095 , i, j) #CZu, CXu, i=CZa, j=Cmq
     # #...debugging....working?!
      real_num = eig_s[1].real
      imag_num = eig_s[1].imag
      real_ft  = eig_test.real
      imag_ft = eig_test.imag
      error_real = abs(real_num - real_ft)
      error_imag = abs(imag_num - real_ft)
      rel_error = abs((abs(eig_s[1])-eig_test))*100/abs(eig_s[1])
      if RMSE(q_out_s_shp,u_shp_p_rate) < min_RMSE_short_period_p_rate and  RMSE(theta_out_s_shp,u_shp_p) < min_RMSE_short_period_p:
         if error_real < error_real_final and error_imag < error_imag_final:
            print('x')
            error_real_final = error_real
            error_imag_final = error_imag
            rel_error_final = rel_error
            CZa_final = i
            #CZadot_final = j
            #CZq_final = k
            #Cmadot_final = l
            Cmq_final = j
      min_RMSE_short_period_p_rate=RMSE(q_out_s_shp,u_shp_p_rate)
      min_RMSE_short_period_p=RMSE(theta_out_s_shp,u_shp_p)
bar.finish()

print("The relative error is:", rel_error_final)
print("CZa = ", CZa_final)
#print("CZadot = ", CZadot_final)
#print("CZq = ", CZq_final)
#print("Cmadot = ", Cmadot_final)
print("Cmq = ", Cmq_final)

"""Ok so these are the initial RMSE for the phugoid:
RMSE_original_phugoid_p_rate = 0.18537027365097794
RMSE_original_phugoid_p = 3.248328645062731"""