# vi: ts=4 expandtab
#
#    Author: Edouard Bonlieu <ebonlieu@ocs.online.net>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License version 3, as
#    published by the Free Software Foundation.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

import functools
import json

from requests.packages.urllib3.poolmanager import PoolManager
import requests

from cloudinit import log as logging
from cloudinit import sources
from cloudinit import url_helper
from cloudinit import util

LOG = logging.getLogger(__name__)

BUILTIN_DS_CONFIG = {
    'metadata_url': 'http://169.254.42.42/conf?format=json',
    'userdata_url': 'http://169.254.42.42/user_data/cloud-init'
}

DEF_MD_RETRIES = 5
DEF_MD_TIMEOUT = 10

class SourceAddressAdapter(requests.adapters.HTTPAdapter):
    """ Adapter for requests to choose the local address to bind to.
    """

    def __init__(self, source_address, **kwargs):
        self.source_address = source_address
        super(SourceAddressAdapter, self).__init__(**kwargs)

    def init_poolmanager(self, connections, maxsize, block=False):
        self.poolmanager = PoolManager(num_pools=connections,
                                       maxsize=maxsize,
                                       block=block,
                                       source_address=self.source_address)

class DataSourceScaleway(sources.DataSource):

    def __init__(self, sys_cfg, distro, paths):
        LOG.debug('Init scw')
        sources.DataSource.__init__(self, sys_cfg, distro, paths)

        self.metadata = {}
        self.ds_cfg = util.mergemanydict([
            util.get_cfg_by_path(sys_cfg, ["datasource", "Scaleway"], {}),
            BUILTIN_DS_CONFIG
        ])

        self.metadata_address = self.ds_cfg['metadata_url']
        self.userdata_address = self.ds_cfg['userdata_url']

        self.retries = self.ds_cfg.get('retries', DEF_MD_RETRIES)
        self.timeout = self.ds_cfg.get('timeout', DEF_MD_TIMEOUT)

    def _get_metadata(self):
        resp = url_helper.readurl(
            self.metadata_address,
            data=None,
            timeout=self.timeout,
            retries=self.retries
        )
        metadata = json.loads(util.decode_binary(resp.contents))
        LOG.debug('metadata downloaded')

        # try to make a request on the first privileged port available
        for port in range(1, 1024):
            try:
                session = requests.Session()
                session.mount(
                    'http://',
                    SourceAddressAdapter(source_address=('0.0.0.0', port))
                )
                resp = url_helper.readurl(
                    self.userdata_address,
                    data=None,
                    timeout=self.timeout,
                    retries=self.retries,
                    session=session
                )
                user_data = util.decode_binary(resp.contents)
                LOG.debug('user-data downloaded')
                return metadata, user_data

            except url_helper.UrlError:  # try next port
                pass

    def get_data(self):
        metadata, metadata['user-data'] = self._get_metadata()
        self.metadata = {
            'id': metadata['id'],
            'hostname': metadata['hostname'],
            'user-data': metadata['user-data'],
            'ssh_public_keys': [
                key['key'] for key in metadata['ssh_public_keys']
            ]
        }
        return True

    @property
    def launch_index(self):
        return None

    def get_instance_id(self):
        return self.metadata['id']

    def get_public_ssh_keys(self):
        return self.metadata['ssh_public_keys']

    def get_hostname(self, fqdn=False, resolve_ip=False):
        return self.metadata['hostname']

    def get_userdata_raw(self):
        return self.metadata['user-data']

    @property
    def availability_zone(self):
        return None

    @property
    def region(self):
        return None

datasources = [
    (DataSourceScaleway, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
]

def get_datasource_list(depends):
    return sources.list_from_depends(depends, datasources)


class Node:
    def __init__(self, data):
        self.data = data
        self.next = None

    def __str(self):
        return self.data

class LinkedList:

    def __init__(self):
        """
           Time complexity: Constant O(1)
           Space complexity: Constant O(1)
        """
        self.size = 0
        self.head = None

    def insert(self, data):
        """
            Time complexity: Constant O(1)
            Space complexity: Constant O(1)
        """
        node = Node(data)
        if self.head:
            node.next = self.head
        self.head = node
        self.size += 1

    def printList(self):
        """
            Time complexity: Linear O(n)
            Space complexity: Constant O(1)
        """
        cur = self.head
        result = ''
        while cur:
            result += f'{cur.data} '
            cur = cur.next
        print(result) if result else print('Empty List!')

    def delete(self, data):
        """
            Time complexity: Linear O(n)
            Space complexity: Constant O(1)
        """
        if self.head:
            if data == self.head.data:
                self.head = self.head.next
                self.size -= 1
            else:
                cur = self.head
                prev = None
                while cur:
                    if cur.data == data:
                        prev.next = cur.next
                        del cur
                        self.size -= 1
                        return
                    prev = cur
                    cur = cur.next

root = LinkedList()
root.printList()
root.insert(10)
root.insert(20)
root.insert(30)
root.printList()
root.delete(20)
root.printList()
root.delete(10)
root.printList()
root.delete(30)
root.printList()
root.insert(40)
root.insert(50)
root.printList()
root.delete(50)
root.printList()
root.delete(40)
root.printList()
root.delete(4000)
root.printList()


"""
@author: Vincent Bonnet
@description : solve a 1D spring - aka damped harmonic oscillator
"""

import numpy as np
import matplotlib.pyplot as plt
from dataclasses import dataclass

SPRING_STIFFNESS = 1.0  # Newtons per meters
SPRING_DAMPING = 0.2   # Netwons per meters per second
INITIAL_POSITION = 2.0 # particle initial position in meters
MASS = 1.0  # particle mass in kg

# time constant
TIME_START = 0.0  # in Second
TIME_END = 30.0   # in Second
DT = 0.1         # in Second
N = round((TIME_END - TIME_START) / DT)

'''
 Classes (State, Derivate, Particle)
'''
@dataclass
class State:
    x = INITIAL_POSITION # position
    v = 0.0 # velocity

@dataclass
class Derivative:
    dx = 0.0 # derivative of x
    dv = 0.0 # derivative of v

@dataclass
class Particle:
    mass = 1.0
    state = State()

'''
  Helper Functions
'''
# internal spring force
def acceleration(state, mass):
    attachement = 0.0
    spring_force = -1.0 * (state.x - attachement) * SPRING_STIFFNESS  # spring force
    spring_force += -1.0 * (state.v * SPRING_DAMPING)  # spring damping
    acceleration = spring_force / mass
    return acceleration

# derivative of x - change in position
def dx(state):
    return state.v

# derivative of v - change in velocity
def dv(state, mass):
    return acceleration(state, mass)

# forward integration - return a new state
def integrate(state, derivative, dt):
    result_state = State()
    result_state.x = state.x + derivative.dx * dt
    result_state.v = state.v + derivative.dv * dt
    return result_state

# compute derivate at the state
def derivate(state, mass):
    result_derivate = Derivative()
    result_derivate.dx = dx(state)
    result_derivate.dv = dv(state, mass)
    return result_derivate

'''
 Integration Functions
'''
def forwardEuler(particle, time, dt):
    k = derivate(particle.state, particle.mass)
    particle.state = integrate(particle.state, k, dt)

def RK2(particle, time, dt):
    s1 = particle.state
    k1 = derivate(s1, particle.mass)
    s2 = integrate(s1, k1, dt * 0.5)
    k2 = derivate(s2, particle.mass)
    particle.state = integrate(particle.state, k2, dt)

def RK4(particle, time, dt):
    s1 = particle.state
    k1 = derivate(s1, particle.mass)
    s2 = integrate(s1, k1, dt * 0.5)
    k2 = derivate(s2, particle.mass)
    s3 = integrate(s1, k2, dt * 0.5)
    k3 = derivate(s3, particle.mass)
    s4 = integrate(s1, k3, dt)
    k4 = derivate(s4, particle.mass)
    k = Derivative()
    k.dx = k1.dx / 6 + k2.dx / 3 + k3.dx / 3 + k4.dx / 6
    k.dv = k1.dv / 6 + k2.dv / 3 + k3.dv / 3 + k4.dv / 6
    particle.state = integrate(particle.state, k, dt)

def semiImplicitEulerV1(particle, time, dt):
    particle.state.v += dv(particle.state, particle.mass) * dt
    particle.state.x += dx(particle.state) * dt

def semiImplicitEulerV2(particle, time, dt):
    particle.state.x += dx(particle.state) * dt
    particle.state.v += dv(particle.state, particle.mass) * dt

def leapFrog(particle, time, dt):
    if (time == TIME_START):
        # compute the velocity at half step which will cause the velocity to be half-step ahead of position
        particle.state.v += dv(particle.state, particle.mass) * dt * 0.5
    particle.state.x += particle.state.v * dt
    particle.state.v += dv(particle.state, particle.mass) * dt

def analyticSolution(particle, time, dt):
    w0 = np.sqrt(SPRING_STIFFNESS/particle.mass)
    y = SPRING_DAMPING / (2 * particle.mass)
    w = np.sqrt(w0 * w0 - y * y)
    a = np.exp(-1.0 * y * time)
    particle.state.x = (INITIAL_POSITION * a * np.cos(w * time));

def main():
    integrators = [(forwardEuler,"xkcd:aqua"),
                   (RK2,"xkcd:plum"),
                   (RK4,"xkcd:teal"),
                   (semiImplicitEulerV1,"xkcd:chartreuse"),
                   (semiImplicitEulerV2,"xkcd:olive"),
                   (leapFrog,"xkcd:green"),
                   (analyticSolution,"xkcd:red" )]

    plt.title('Single Damped Harmonic Oscillator')
    plt.xlabel('time(t)')
    plt.ylabel('position(x)')

    # integrators Loop
    time_samples = np.linspace(TIME_START, TIME_END, num=N, endpoint=False)
    position_samples = np.zeros(N)
    for integrator in integrators:
        function = integrator[0]
        plot_colour = integrator[1]
        particle = Particle();

        for i in range(N):
            position_samples[i] = particle.state.x
            function(particle, time_samples[i], DT)

        plt.plot(time_samples, position_samples, color=plot_colour, label=function.__name__)

    # display result
    plt.legend(bbox_to_anchor=(1, 1), loc=2)
    plt.show()

if __name__ == '__main__':
    main()


# for colab
# from google.colab import auth
from private_constants import PrivateConstants

import os
from googleapiclient.discovery import build
from googleapiclient.http import MediaIoBaseDownload
from google.cloud import storage

class GCS:
    def list_files_from_gcs(bucket_name, path):
        gcs_service = storage.Client(PrivateConstants.PROJECT_NAME)
        bucket = gcs_service.bucket(bucket_name)
        file_names = []
        blobs = bucket.list_blobs(prefix=path)
        for blob in blobs:
            file_names.append(blob.name)
        return file_names

    def download_files_from_gcs(bucket_name, path, to_dir):
        gcs_service = storage.Client(PrivateConstants.PROJECT_NAME)
        bucket = gcs_service.bucket(bucket_name)
        blobs = bucket.list_blobs(prefix=path)
        for blob in blobs:
            file_name = blob.name.rsplit('/', 1)[-1]
            if file_name == "":
                continue
            blob.download_to_filename(os.path.join(to_dir, file_name))
        print("Download to {}/{} complete".format(bucket_name, path))
        return True

    def upload_to_gcs(bucket_name, from_file, to_file):
        gcs_service = storage.Client(PrivateConstants.PROJECT_NAME)
        bucket = gcs_service.bucket(bucket_name)
        blob = bucket.blob(to_file)
        blob.upload_from_filename(from_file)
        print("Upload to {}/{} complete".format(bucket_name, to_file))
        return True

    def download_file_from_gcs(bucket_name, from_file, to_file):
        gcs_service = storage.Client(PrivateConstants.PROJECT_NAME)
        bucket = gcs_service.bucket(bucket_name)
        blob = bucket.blob(from_file)
        blob.download_to_filename(to_file)
        print("Download from {}/{} complete".format(bucket_name, from_file))
        return True

    # DEPRECATED
    def get_from_gcs(bucket_name, from_file, to_file):
        # Create the service client.
        gcs_service = build('storage', 'v1')

        with open(to_file, 'wb') as f:
            request = gcs_service.objects().get_media(bucket=bucket_name,
                                                      object=from_file)
            media = MediaIoBaseDownload(f, request)

            done = False
            while not done:
                # _ is a placeholder for a progress object that we ignore.
                # (Our file is small, so we skip reporting progress.)
                _, done = media.next_chunk()

        print('Download complete')

    # DEPRECATED
    def store_in_gcs(bucket_name, from_file, to_file):
        # Create the service client.
        gcs_service = build('storage', 'v1')

        #   name for name in os.listdir(DIR) if os.path.isfile(os.path.join(DIR, name))

        with open(to_file, 'wb') as f:
            request = gcs_service.objects().get_media(bucket=bucket_name,
                                                      object=from_file)
            media = MediaIoBaseDownload(f, request)

            done = False
            while not done:
                # _ is a placeholder for a progress object that we ignore.
                # (Our file is small, so we skip reporting progress.)
                _, done = media.next_chunk()

        print('Upload complete')


import matplotlib as mpl
mpl.use('agg')
from lauren import *
from radial_data import *
import matplotlib.pyplot as plt

#fn="/Users/laurennc/data/RyanSims/r0058_l10/redshift0058"
fn="/hpc/astrostats/astro/users/lnc2115/Ryan/r0058_l10/redshift0058"
pf = load(fn, file_style="%s.grid.cpu%%04i") # load data

val = 2.37163264206e-23
pos = [0.40328598,0.47176743,0.46131516]

#fields = ["HI_Number_Density","HAlpha_Emissivity","HAlpha_Emissivity_R","HAlpha_Voort_R"]
fields = ["HAlpha_Emissivity_R","HAlpha_Voort_R"]

proj = pf.h.proj(0,fields)
width = 200./pf['kpc']
res = [500,500]
frb = proj.to_frb(width,res,center=pos)

xL = np.arange(-250,250)*0.4
xL, yL = np.meshgrid(xL,xL)

#rp_Arc = radial_data(frb['HAlpha_Emissivity'],x=xL,y=yL)
rp_Ral = radial_data(frb['HAlpha_Emissivity_R'],x=xL,y=yL)
rp_Vor = radial_data(frb['HAlpha_Voort_R'],x=xL,y=yL)
#rp_hdens = radial_data(frb['HI_Number_Density'],x=xL,y=yL)

#thisindex = (rp_Arc.r/250. >0.01)
#plt.plot((rp_Arc.r/250.)[thisindex],rp_Arc.mean[thisindex])
#plt.yscale('log')
#plt.xscale('log')
#plt.xlabel('Radius / Rvir')
#plt.ylabel('Surface Brightness (ergs s^-1 cm^-2 arcsec^-2)')
#plt.savefig('rp_haArc_cloudy_smooth.png')
#plt.close()

thisindex = (rp_Ral.r/250. >0.01)
plt.plot((rp_Ral.r/250.)[thisindex],rp_Ral.mean[thisindex])
plt.yscale('log')
plt.xscale('log')
plt.xlabel('Radius / Rvir')
plt.ylabel('Surface Brightness (R)')
plt.xlim(10**-2,1)
plt.ylim(10**-5,10)
plt.savefig('rp_haR_cloudy_smooth.png')
plt.close()

thisindex = (rp_Vor.r/250. >0.01)
plt.plot((rp_Vor.r/250.)[thisindex],rp_Vor.mean[thisindex])
plt.yscale('log')
plt.xscale('log')
plt.xlabel('Radius / Rvir')
plt.ylabel('Surface Brightness (R)')
plt.xlim(10**-2,1)
plt.ylim(10**-5,10)
plt.savefig('rp_haVoortR_cloudy_smooth.png')
plt.close()

#thisindex = (rp_hdens.r/250. >0.01)
#plt.plot((rp_hdens.r/250.)[thisindex],rp_hdens.mean[thisindex])
#plt.yscale('log')
#plt.xscale('log')
#plt.xlabel('Radius / Rvir')
#plt.ylabel('HI Column Density (cm^-2)')
#plt.savefig('rp_hdens_cloudy_smooth.png')
#plt.close()


t = int(input("qnt inicial de tambaquis:"))
pc = int(input("percentual de crescimento:"))/100
qr = int(input("quantia retirada do tanque"))
a = 0
while(t > 0 and t < 12000):
	t = t + t * pc - qr
	a = a + 1
if(t<=0):
	print("EXTINCAO")
else:
	print("LIMITE")
print(a)

5 - Notebooks e Data/1 - Análises numéricas/Arquivos David/Atualizados/logDicas-master/data/2019-1/226/users/4157/codes/1723_2506.py


# Import data types
from pyspark.sql.types import *

# Load a text file and convert each line to a Row.
lines = sc.textFile("/data/spark/people.txt")
parts = lines.map(lambda l: l.split(","))
# Each line is converted to a tuple.
people = parts.map(lambda p: (p[0], p[1].strip()))

# The schema is encoded in a string.
schemaString = "name age"

fields = [StructField(field_name, StringType(), True) for field_name in schemaString.split()]
schema = StructType(fields)

# Apply the schema to the RDD.
schemaPeople = spark.createDataFrame(people, schema)
schemaPeople.show()


"""Utility Widgets"""
from itertools import groupby

from django.forms.models import ModelChoiceField, ModelChoiceIterator

class GroupedModelChoiceIterator(ModelChoiceIterator):
    """ModelChoiceField which groups objects by their verbose_name"""

    def __iter__(self):
        if self.field.empty_label is not None:
            yield ("", self.field.empty_label)
        queryset = self.queryset
        # Can't use iterator() when queryset uses prefetch_related()
        if not queryset._prefetch_related_lookups:
            queryset = queryset.iterator()
        # We can't use DB-level sorting as we sort by subclass
        queryset = sorted(queryset, key=lambda x: x._meta.verbose_name)
        for group, objs in groupby(queryset, key=lambda x: x._meta.verbose_name):
            yield (group, [self.choice(obj) for obj in objs])

class GroupedModelChoiceField(ModelChoiceField):
    """ModelChoiceField which groups objects by their verbose_name"""

    iterator = GroupedModelChoiceIterator


import re
sam_file = "030.sam"
fasta_file = "030_from_sam.fasta"
bffr = ""

compl = {'A':'T', 'G':'C', 'C':'G', 'T':'A', 'Y':'R', 'R':'Y', 'W':'W', 'S':'S', \
'K':'M', 'M':'K', 'D':'H', 'V':'B', 'H':'D', 'B':'V', 'X':'X', 'N':'N', '-':'-'}

with open(sam_file, "r") as fhandle:
	for line in fhandle:
		if not line.startswith("@"):
			bits = re.split("\s+", line)
			if bits[1] == '0':
				bffr += ">{0}\n{1}\n".format(bits[0], bits[9])
			elif bits[1] == '16':
				reverse = bits[9][::-1]
				co = "".join([compl[x] for x in reverse.upper()])
				bffr += ">{0}\n{1}\n".format(bits[0], co)
		if len(bffr) > 10000:
			with open(fasta_file, "a") as ihandle:
				ihandle.write(bffr)
			bffr = ""
with open(fasta_file, "a") as ihandle:
	ihandle.write(bffr)

exit()


#
# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from com.nvidia.spark.examples.mortgage.consts import *
from pyspark.sql.functions import *
from pyspark.sql.types import *
from sys import exit

get_quarter = udf(lambda path: path.split(r'.')[0].split('_')[-1], StringType())
standardize_name = udf(lambda name: name_mapping.get(name), StringType())

def load_data(spark, paths, schema, args, extra_csv_opts={}):
    reader = (spark
        .read
        .format(args.format)
        .option('asFloats', args.asFloats)
        .option('maxRowsPerChunk', args.maxRowsPerChunk))
    if args.format == 'csv':
        (reader
            .schema(schema)
            .option('delimiter', '|')
            .option('header', args.hasHeader))
        for k, v in extra_csv_opts.items():
            reader.option(k, v)
    return reader.load(paths)

def prepare_performance(spark, args):
    extra_csv_options = {
        'nullValue': '',
        'parserLib': 'univocity',
    }
    paths = extract_paths(args.dataPaths, 'perf::')
    performance = (load_data(spark, paths, performance_schema, args, extra_csv_options)
        .withColumn('quarter', get_quarter(input_file_name()))
        .withColumn('timestamp', to_date(col('monthly_reporting_period'), 'MM/dd/yyyy'))
        .withColumn('timestamp_year', year(col('timestamp')))
        .withColumn('timestamp_month', month(col('timestamp'))))

    aggregation = (performance
        .select(
            'quarter',
            'loan_id',
            'current_loan_delinquency_status',
            when(col('current_loan_delinquency_status') >= 1, col('timestamp'))
                .alias('delinquency_30'),
            when(col('current_loan_delinquency_status') >= 3, col('timestamp'))
                .alias('delinquency_90'),
            when(col('current_loan_delinquency_status') >= 6, col('timestamp'))
                .alias('delinquency_180'))
        .groupBy('quarter', 'loan_id')
        .agg(
            max('current_loan_delinquency_status').alias('delinquency_12'),
            min('delinquency_30').alias('delinquency_30'),
            min('delinquency_90').alias('delinquency_90'),
            min('delinquency_180').alias('delinquency_180'))
        .select(
            'quarter',
            'loan_id',
            (col('delinquency_12') >= 1).alias('ever_30'),
            (col('delinquency_12') >= 3).alias('ever_90'),
            (col('delinquency_12') >= 6).alias('ever_180'),
            'delinquency_30',
            'delinquency_90',
            'delinquency_180'))

    months = spark.createDataFrame(range(12), IntegerType()).withColumnRenamed('value', 'month_y')
    to_join = (performance
        .select(
            'quarter',
            'loan_id',
            'timestamp_year',
            'timestamp_month',
            col('current_loan_delinquency_status').alias('delinquency_12'),
            col('current_actual_upb').alias('upb_12'))
        .join(aggregation, ['loan_id', 'quarter'], 'left_outer')
        .crossJoin(months)
        .select(
            'quarter',
            floor(
                (col('timestamp_year') * 12 + col('timestamp_month') - 24000 - col('month_y')) / 12
            ).alias('josh_mody_n'),
            'ever_30',
            'ever_90',
            'ever_180',
            'delinquency_30',
            'delinquency_90',
            'delinquency_180',
            'loan_id',
            'month_y',
            'delinquency_12',
            'upb_12')
        .groupBy(
            'quarter',
            'loan_id',
            'josh_mody_n',
            'ever_30',
            'ever_90',
            'ever_180',
            'delinquency_30',
            'delinquency_90',
            'delinquency_180',
            'month_y')
        .agg(
            max('delinquency_12').alias('delinquency_12'),
            min('upb_12').alias('upb_12'))
        .withColumn(
            'timestamp_year',
            floor((24000 + (col('josh_mody_n') * 12) + (col('month_y') - 1)) / 12))
        .withColumn(
            'timestamp_month_tmp',
            (24000 + (col('josh_mody_n') * 12) + col('month_y')) % 12)
        .withColumn(
            'timestamp_month',
            when(col('timestamp_month_tmp') == 0, 12).otherwise(col('timestamp_month_tmp')))
        .withColumn(
            'delinquency_12',
            ((col('delinquency_12') > 3).cast('int') + (col('upb_12') == 0).cast('int')))
        .drop('timestamp_month_tmp', 'josh_mody_n', 'month_y'))

    return (performance
        .join(to_join, ['quarter', 'loan_id', 'timestamp_year', 'timestamp_month'], 'left')
        .drop('timestamp_year', 'timestamp_month'))

def prepare_acquisition(spark, args):
    return (load_data(spark, extract_paths(args.dataPaths, 'acq::'), acquisition_schema, args)
        .withColumn('quarter', get_quarter(input_file_name()))
        .withColumn('seller_name', standardize_name(col('seller_name'))))

def extract_paths(paths, prefix):
    results = [ path[len(prefix):] for path in paths if path.startswith(prefix) ]
    if not results:
        print('-' * 80)
        print('Usage: {} data path required'.format(prefix))
        exit(1)
    return results

def etl(spark, args):
    performance = prepare_performance(spark, args)
    acquisition = prepare_acquisition(spark, args)
    return (performance
        .join(acquisition, ['loan_id', 'quarter'], 'left_outer')
        .select(
            [(md5(col(x)) % 100).alias(x) for x in categorical_columns]
            + [col(x) for x in numeric_columns])
        .withColumn('delinquency_12', when(col('delinquency_12') > 0, 1).otherwise(0))
        .na
        .fill(0))


from distutils.core import setup, Extension
import numpy as np

module1 = Extension('p2p',
                    sources = ['py2petsc.c'])

setup (name = 'p2p',
       version = '1.0',
       description = 'This is a package to call PetSC from Python',
       ext_modules = [module1],
       include_dirs = [np.get_include()])


import datetime
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_absolute_error
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import r2_score
from sklearn.metrics import mean_squared_error
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import RandomizedSearchCV
from joblib import dump, load
from . import utils
import os

def createFeatures(dfTripsPerDay):

    ###Feature creation
    #Get the date of yesterday and one week ago
    dfTripsPerDay['date_yesterday'] = pd.DatetimeIndex(dfTripsPerDay['date']) - pd.DateOffset(1)
    dfTripsPerDay['date_oneWeekAgo'] = pd.DatetimeIndex(dfTripsPerDay['date']) - pd.DateOffset(7)

    #Get the number of trips for this dates
    tripsLastDay = []
    tripsOneWeekAgo = []

    #Save unique dates
    lastDay = list(dfTripsPerDay['date_yesterday'].unique())
    oneWeekEarlier = list(dfTripsPerDay['date_oneWeekAgo'].unique())

    #Try/except is necessary, because of the outlier drop
    for date in lastDay:
        try:
            valueDay = dfTripsPerDay[dfTripsPerDay['date'] == date]['tripsPerDay'].values[0]
        except:
            valueDay = None

        tripsLastDay.append(valueDay)

    for date in oneWeekEarlier:
        try:
            valueWeek = dfTripsPerDay[dfTripsPerDay['date'] == date]['tripsPerDay'].values[0]
        except:
            valueWeek = None

        tripsOneWeekAgo.append(valueWeek)

    #Fill in created features
    dfTripsPerDay['tripsLastDay'] = tripsLastDay
    dfTripsPerDay['tripsOneWeekAgo'] = tripsOneWeekAgo

    #Get the mean for fill of nans
    avgTripsPerDay = dfTripsPerDay['tripsPerDay'].mean()

    #Fill nans with mean
    dfTripsPerDay['tripsLastDay'].fillna(avgTripsPerDay,inplace=True)
    dfTripsPerDay['tripsOneWeekAgo'].fillna(avgTripsPerDay,inplace=True)

    #Drop Dates and Features with a low correlation
    dfTripsPerDay['isTerm'] = dfTripsPerDay.apply(isTerm, axis=1)
    dfTripsPerDay.dropna(inplace=True)
    dfTripsPerDay.drop(['date','date_yesterday','date_oneWeekAgo'],axis=1,inplace=True)
    dfTripsPerDay.drop(['day','month'],axis=1,inplace=True)
    dfTripsPerDay.drop(['temperatureMIN','temperatureMAX'],axis=1,inplace=True)

    return dfTripsPerDay

# Trains a model to predict the number of trips on the next day
def retrainModel_NumberOfTrips(dfTripsPerDay, optimalHyperparameterTest):

    ###First filter for outliers
    #Get standart deviation of trips per day
    std = dfTripsPerDay['tripsPerDay'].std()

    #For each month
    for i in range(1,13):
        #Get the average number of trips per month
        df = dfTripsPerDay[dfTripsPerDay['month']==i]
        avg = df['tripsPerDay'].mean()

        #Only use days where the average is within the standartdeviation
        row = df[(df['tripsPerDay'] <= avg+std)&(df['tripsPerDay'] >= avg-std)]

        #Combine the data for the months again
        if i == 1:
            dfTripsPerDayFilterd = row
        else:
            dfTripsPerDayFilterd = pd.concat([dfTripsPerDayFilterd,row])

    #Create features
    dfTripsPerDayFilterd = createFeatures(dfTripsPerDayFilterd)

    #Make a train test split
    Y = dfTripsPerDayFilterd['tripsPerDay']
    X = dfTripsPerDayFilterd.drop('tripsPerDay',axis=1).values
    x_train, x_test, y_train, y_test = train_test_split(X, Y, test_size=0.3, random_state=20)

    #Scale the inputdata
    sscaler = StandardScaler()
    x_train = sscaler.fit_transform(x_train)
    x_test = sscaler.transform(x_test)

    #Sacle the output data
    sscalerY = StandardScaler()
    y_train = sscalerY.fit_transform(y_train.values.reshape(-1, 1))
    y_test = sscalerY.transform(y_test.values.reshape(-1, 1))
    #Find the optimal Hyperparameter by testing multiple combinations using RandomizedSearchCV
    if optimalHyperparameterTest == True:

        n_estimators = [int(x) for x in np.linspace(start = 100, stop = 1500, num = 10)]
        max_features = ['auto', 'sqrt']
        max_depth = [int(x) for x in np.linspace(10, 110, num = 11)]
        max_depth.append(None)
        min_samples_split = [2, 5, 10]
        min_samples_leaf = [1, 2, 4]
        bootstrap = [True, False]
        random_grid = {'n_estimators': n_estimators,
                'max_features': max_features,
                'max_depth': max_depth,
                'min_samples_split': min_samples_split,
                'min_samples_leaf': min_samples_leaf,
                'bootstrap': bootstrap}

        rf = RandomForestRegressor()
        rf_random = RandomizedSearchCV(estimator = rf, param_distributions = random_grid, n_iter = 150, cv = 3, verbose=2, random_state=45, n_jobs = -1)
        rf_random.fit(x_train,y_train)

        #Take the best values
        reg = RandomForestRegressor(n_jobs=-1,
                            n_estimators = rf_random.best_params_['n_estimators'],
                            min_samples_split = rf_random.best_params_['min_samples_split'],
                            min_samples_leaf = rf_random.best_params_['min_samples_leaf'],
                            max_features = rf_random.best_params_['max_features'],
                            max_depth = rf_random.best_params_['max_depth'],
                            bootstrap = rf_random.best_params_['bootstrap']
                            )

    else:
        #Else use default
        reg = RandomForestRegressor(n_jobs=-1,n_estimators=1000)

    #Train the model
    reg.fit(x_train,y_train)

    #Predict the test data and rescale them
    y_pred_test = reg.predict(x_test)
    y_pred_test = sscalerY.inverse_transform(y_pred_test)
    y_test = sscalerY.inverse_transform(y_test)

    #Evaluate the model based on test set
    errTest = mean_absolute_error(y_test, y_pred_test)
    err_mseTest = mean_squared_error(y_test, y_pred_test)
    err_r2Test = r2_score(y_test, y_pred_test)

    #Predict the train data and rescale them
    y_pred_train = reg.predict(x_train)
    y_pred_train = sscalerY.inverse_transform(y_pred_train)
    y_train = sscalerY.inverse_transform(y_train)

    #Evaluate the model based on train set
    errTrain = mean_absolute_error(y_train, y_pred_train)
    err_mseTrain = mean_squared_error(y_train, y_pred_train)
    err_r2Train = r2_score(y_train, y_pred_train)

    #Print the results
    print("Test  :  ","MAE: ",errTest,"MSE: ",err_mseTest,"R^2: ",err_r2Test)
    print("Train :  ","MAE: ",errTrain,"MSE: ",err_mseTrain,"R^2: ",err_r2Train)

    #Visualize the Prediction
    day = []
    for i in range(len(y_test)):
        day.append(i)

    plt.plot(day, y_test,label="Number of trips")
    plt.plot(day,y_pred_test,label="Predicted number of trips")
    plt.title("Visualization of the Prediction")
    plt.xlabel("Date")
    plt.ylabel("Number of Trips")
    plt.legend(loc="upper left")

    plt.tight_layout()
    plt.show()

    #Retrain the model on all available data

    #Scale the Data
    sscaler = StandardScaler()
    xScaled = sscaler.fit_transform(X)
    sscalerY = StandardScaler()
    yScaled = sscalerY.fit_transform(Y.values.reshape(-1, 1))

    reg.fit(xScaled,yScaled)
    #Save the model
    path = os.path.join(utils.get_ml_path(), "numberOfTrips/randomForestRegressor_NumberOfTrips.pkl")
    dump(reg,path)

    # Save the scaler
    pathScaler = os.path.join(utils.get_ml_path(), "numberOfTrips/randomForestRegressor_NumberOfTripsScaler.bin")
    dump(sscaler,pathScaler)

    pathScalerY = os.path.join(utils.get_ml_path(), "numberOfTrips/randomForestRegressor_NumberOfTripsScalerY.bin")
    dump(sscalerY,pathScalerY)

#Loads the saved ml-model and scaler
def loadModel_NumberOfTrips():
    #Load random forest regressor
    path = os.path.join(utils.get_ml_path(), "numberOfTrips/randomForestRegressor_NumberOfTrips.pkl")
    rfr = load(path , mmap_mode ='r')
    #Load scaler for input data
    pathScaler = os.path.join(utils.get_ml_path(), "numberOfTrips/randomForestRegressor_NumberOfTripsScaler.bin")
    sscaler = load(pathScaler)
    #Load scaler for target feature
    pathScalerY = os.path.join(utils.get_ml_path(), "numberOfTrips/randomForestRegressor_NumberOfTripsScalerY.bin")
    sscalerY = load(pathScalerY)

    return rfr,sscaler, sscalerY

#Predicts the number of trips for the next day
def predict_NumberOfTrips(dfInput, model, sscaler, sscalerY):

    #Mean values for compare
    meanOfAllValues = 1672
    meanOfPrevoiusMonth = 2921

    #Create inputs dataframe and scale it
    df = dfInput.copy()
    #Create necessary features
    features = createFeatures(df)
    #Save the true values to evaluate later
    goal = features['tripsPerDay']
    features.drop('tripsPerDay',inplace=True,axis=1,errors='ignore')
    featureValues = features.values
    xScaled = sscaler.transform(featureValues)

    #Predict and inverse transformation
    prediction = model.predict(xScaled)
    prediction = sscalerY.inverse_transform(prediction)

    # Save data
    path = os.path.join(utils.get_prediction_path(), "output/NumberOfTripPrediction.csv")
    df['tripsPerDay'] = prediction
    df.to_csv(path, index=False)

    print("Prediction is saved to csv --> output/NumberOfTripPrediction.csv")

    #Evaluate the prediction
    errTest = mean_absolute_error(goal, prediction)
    err_mseTest = mean_squared_error(goal, prediction)
    err_r2Test = r2_score(goal, prediction)

    print("Test  :  ","MAE: ",errTest,"MSE: ",err_mseTest,"R^2: ",err_r2Test)

    #Compare with mean methods
    allPreviousAvg = [meanOfAllValues] * len(goal)
    errAllPreviousAvg = mean_absolute_error(goal, allPreviousAvg)
    err_mseAllPreviousAvg = mean_squared_error(goal, allPreviousAvg)

    previousMonthAvg = [meanOfPrevoiusMonth] * len(goal)
    errPreviousMonthAvg = mean_absolute_error(goal, previousMonthAvg)
    err_msePreviousMonthAvg = mean_squared_error(goal, previousMonthAvg)

    print("Compare to prediction by average (last 6 months):  ","MAE: ",errAllPreviousAvg," MSE: ", err_mseAllPreviousAvg)
    print("Compare to prediction by average (last month):  ","MAE: ",errPreviousMonthAvg," MSE: ", err_msePreviousMonthAvg)

    # Plot data
    # TODO Fix legend and axis
    plt.plot(dfInput['date'],prediction, label="Predicted number of trips")
    plt.plot(dfInput['date'],goal, label="Number of trips")
    plt.title("Visualization of the Prediction")
    plt.xlabel("Date")
    plt.xticks(rotation=45)
    plt.ylabel("Number of Trips")
    plt.legend(loc="upper left")

    plt.tight_layout()
    plt.show()

    return df

#Checks if uni is open
def isTerm(row):
    value = -1
    if datetime.datetime(2019, 1, 7) <= row['date'] <= datetime.datetime(2019, 2, 15):
        value = 1
    elif datetime.datetime(2019, 4, 15) <= row['date'] <= datetime.datetime(2019, 7, 19):
        value = 1
    elif datetime.datetime(2019, 10, 14) <= row['date'] <= datetime.datetime(2019, 12, 20):
        value = 1
    else:
        value = 0
    return value


import os
import subprocess
from unittest import mock

from . import BuilderTest, MockPackage, through_json
from .. import mock_open_log

from mopack.builders import Builder
from mopack.builders.custom import CustomBuilder
from mopack.path import Path
from mopack.shell import ShellArguments
from mopack.origins.sdist import DirectoryPackage

class TestCustomBuilder(BuilderTest):
    builder_type = CustomBuilder

    def check_build(self, builder, build_commands=None, *, pkg=None):
        if pkg is None:
            pkg = MockPackage(srcdir=self.srcdir, _options=self.make_options())
        if build_commands is None:
            builddir = os.path.join(self.pkgdir, 'build', builder.name)
            build_commands = [i.fill(srcdir=self.srcdir, builddir=builddir)
                              for i in builder.build_commands]

        with mock_open_log() as mopen, \
             mock.patch('mopack.builders.custom.pushd'), \
             mock.patch('subprocess.run') as mcall:
            builder.build(self.metadata, pkg)
            mopen.assert_called_with(os.path.join(
                self.pkgdir, 'logs', 'foo.log'
            ), 'a')
            for line in build_commands:
                mcall.assert_any_call(
                    line, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                    universal_newlines=True, check=True, env={}
                )

    def test_basic(self):
        builder = self.make_builder('foo', build_commands=[
            'configure', 'make',
        ])
        self.assertEqual(builder.name, 'foo')
        self.assertEqual(builder.build_commands, [
            ShellArguments(['configure']),
            ShellArguments(['make']),
        ])
        self.assertEqual(builder.deploy_commands, [])
        self.check_build(builder)

    def test_build_list(self):
        builder = self.make_builder('foo', build_commands=[
            ['configure', '--foo'], ['make', '-j2']
        ])
        self.assertEqual(builder.name, 'foo')
        self.assertEqual(builder.build_commands, [
            ShellArguments(['configure', '--foo']),
            ShellArguments(['make', '-j2']),
        ])
        self.assertEqual(builder.deploy_commands, [])
        self.check_build(builder)

    def test_path_objects(self):
        builder = self.make_builder('foo', build_commands=[
            'configure $srcdir/build',
            ['make', '-C', '$builddir'],
        ])
        self.assertEqual(builder.name, 'foo')
        self.assertEqual(builder.build_commands, [
            ShellArguments(['configure', (Path('', 'srcdir'), '/build')]),
            ShellArguments(['make', '-C', Path('', 'builddir')]),
        ])
        self.assertEqual(builder.deploy_commands, [])
        self.check_build(builder, build_commands=[
            ['configure', self.srcdir + '/build'],
            ['make', '-C', os.path.join(self.pkgdir, 'build', 'foo')],
        ])

    def test_deploy(self):
        pkg = MockPackage(srcdir=self.srcdir, _options=self.make_options())
        builder = self.make_builder(pkg, build_commands=['make'],
                                    deploy_commands=['make install'])
        self.assertEqual(builder.name, 'foo')
        self.assertEqual(builder.build_commands, [
            ShellArguments(['make']),
        ])
        self.assertEqual(builder.deploy_commands, [
            ShellArguments(['make', 'install']),
        ])
        self.check_build(builder)

        with mock_open_log() as mopen, \
             mock.patch('mopack.builders.custom.pushd'), \
             mock.patch('subprocess.run') as mcall:
            builder.deploy(self.metadata, pkg)
            mopen.assert_called_with(os.path.join(
                self.pkgdir, 'logs', 'deploy', 'foo.log'
            ), 'a')
            mcall.assert_called_with(
                ['make', 'install'], stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT, universal_newlines=True,
                check=True, env={}
            )

    def test_cd(self):
        builder = self.make_builder('foo', build_commands=[
            'configure $srcdir/build',
            'cd $builddir',
            'make',
        ])
        self.assertEqual(builder.name, 'foo')
        self.assertEqual(builder.build_commands, [
            ShellArguments(['configure', (Path('', 'srcdir'), '/build')]),
            ShellArguments(['cd', Path('', 'builddir')]),
            ShellArguments(['make']),
        ])

        with mock.patch('os.chdir') as mcd:
            builddir = os.path.join(self.pkgdir, 'build', 'foo')
            self.check_build(builder, build_commands=[
                ['configure', self.srcdir + '/build'],
                ['make'],
            ])
            mcd.assert_called_once_with(builddir)

    def test_cd_invalid(self):
        pkg = MockPackage(srcdir=self.srcdir, _options=self.make_options())
        builder = self.make_builder(pkg, build_commands=['cd foo bar'])

        with mock_open_log(), \
             mock.patch('mopack.builders.custom.pushd'), \
             self.assertRaises(RuntimeError):
            builder.build(self.metadata, pkg)

    def test_clean(self):
        pkg = MockPackage(srcdir=self.srcdir, _options=self.make_options())
        builder = self.make_builder(pkg, build_commands=['make'])
        builddir = os.path.join(self.pkgdir, 'build', 'foo')

        with mock.patch('shutil.rmtree') as mrmtree:
            builder.clean(self.metadata, pkg)
            mrmtree.assert_called_once_with(builddir, ignore_errors=True)

    def test_linkage(self):
        opts = self.make_options()
        pkg = DirectoryPackage('foo', path=self.srcdir, build={
            'type': 'custom', 'build_commands': ['make'],
        }, linkage='pkg_config', _options=opts, config_file=self.config_file)
        pkg.get_linkage(self.metadata, None)

    def test_rehydrate(self):
        opts = self.make_options()
        builder = CustomBuilder(MockPackage('foo', _options=opts),
                                build_commands=['make'])
        data = through_json(builder.dehydrate())
        self.assertEqual(builder, Builder.rehydrate(data, _options=opts))

    def test_upgrade(self):
        opts = self.make_options()
        data = {'type': 'custom', '_version': 0, 'name': 'foo',
                'build_commands': [], 'deploy_commands': None}
        with mock.patch.object(CustomBuilder, 'upgrade',
                               side_effect=CustomBuilder.upgrade) as m:
            pkg = Builder.rehydrate(data, _options=opts)
            self.assertIsInstance(pkg, CustomBuilder)
            m.assert_called_once()


import json

from django.core.paginator import Paginator
from django.shortcuts import render
from django.http import HttpResponse
from .procedures import UsersDB

# Create your views here.
def users_list(request):
    udb = UsersDB()
    if request.method == 'GET':
        order_by = 'name'

        if request.GET.get('limit', False):
            limit = int(request.GET.get('limit'))
            limit = limit if limit >= 10 and limit <= 20 else 10
        else:
            limit = 10

        if request.GET.get('page', False):
            page = int(request.GET.get('page'))
        else:
            page = 1

        if request.GET.get('search', False):
            users_filter = request.GET.get('search')
        else:
            users_filter = None

        if users_filter:
            users_list = udb.search_users([users_filter, order_by])
        else:
            users_list = udb.get_users_list([order_by])
        paginator = Paginator(users_list, limit)
        users_list = paginator.page(page)
        del udb
        return render(request, 'users_list.html',
                      {'users': users_list, 'limit': limit,
                       'users_filter': users_filter})

    if request.method == 'POST' and request.is_ajax():
        if request.POST.get('remove_user', False):
            user_id = request.POST.get('remove_user')
            udb.remove_user([user_id])
            del udb
            return HttpResponse(status=204)

def courses_list(request):
    udb = UsersDB()
    courses_list = udb.get_courses_list()
    del udb
    return render(request, 'courses_list.html', {'courses': courses_list})

def create_user(request):
    if request.method == 'POST' and request.is_ajax():
        if request.POST.get('name', False)\
           and request.POST.get('email', False):
            user_name = request.POST.get('name')
            user_email = request.POST.get('email')

            user_phone = request.POST.get('phone_no')
            user_phone = user_phone if user_phone else None

            user_mobile = request.POST.get('mobile_no')
            user_mobile = user_mobile if user_mobile else None

            user_status = request.POST.get('status')
            user_status = True if user_status == 'active' else False

            udb = UsersDB()
            udb.add_user([user_name, user_email, user_status,
                          user_phone, user_mobile])
            del udb
            return HttpResponse(status=204)
    return render(request, 'create_user.html')

def change_user(request, uid):
    udb = UsersDB()
    if request.method == 'POST' and request.is_ajax():
        if request.POST.get('name', False)\
           and request.POST.get('email', False):
            user_name = request.POST.get('name')
            user_email = request.POST.get('email')

            user_phone = request.POST.get('phone_no')
            user_phone = user_phone if user_phone else None

            user_mobile = request.POST.get('mobile_no')
            user_mobile = user_mobile if user_mobile else None

            user_status = request.POST.get('status')
            user_status = True if user_status == 'active' else False
            udb.change_user([uid, user_name, user_email, user_status,
                             user_phone, user_mobile])
            del udb
            return HttpResponse(status=204)

        if request.POST.get('1'):
            udb.remove_users_course([uid])
            for i in request.POST.items():
                if request.POST.get(i[0]) == 'true':
                    udb.add_users_course([uid, i[0]])

            return HttpResponse(status=204)
        return HttpResponse(status=204)

    if request.method == 'GET':
        user = udb.get_user([uid])
        courses = udb.get_courses_list()
        user_courses = udb.get_users_courses([uid])

        show_courses = []

        courses_status = []

        for course in courses:
            if course not in user_courses\
               and course not in show_courses:
                course_status = list(course)
                course_status.append(False)
                courses_status.append(course_status)
                show_courses.append(course)
            elif course not in show_courses:
                course_status = list(course)
                course_status.append(True)
                courses_status.append(course_status)
        del udb
        return render(request, 'change_user.html',
                      {'user': user, 'courses': show_courses,
                       'user_courses': user_courses,
                       'js_courses': json.dumps(courses_status)})


import matplotlib.pyplot as plt
import numpy as np
from drawnow import drawnow
from matplotlib import pyplot
from matplotlib.animation import FuncAnimation

import time
import math

figure = pyplot.figure(figsize=(12,15))
ax1 = figure.add_subplot(1,1,1)
ideas={}
axes={}

def initIdeas():
    f=open("../ProjectFiles/graphs/ideas.txt",'r+')
    lines=f.readlines()
    nbrColors=len(lines)
    incrementValue=math.floor((2**24)/nbrColors)
    color=0

    for line in lines:
        #Init Data
        line=line.replace("\n","")
        details=line.split(":")
        ideas[details[0]]=(int(details[1]),'%06X' % color)
        axes[details[0]]=np.array([int(details[1])])

       # axes[details[0]]=(np.array([0],[int(details[1])]))

        color+=incrementValue

    print(ideas)

def getIdeaColor(idea):
    ideaInformations=ideas.get(idea)
    return ideaInformations[1]

def updateFigure():

    for idea,XYaxe in axes.items():
        print(XYaxe)
        plt.plot(XYaxe,'o-',color="#"+getIdeaColor(idea),label=idea,linewidth=2,figure=figure)
        plt.axis(xmin=0)

def decNbrIdea(idea):
    details=ideas[idea]

    ideas[idea]=(details[0]-1,details[1])

    #details[0]+=1
    #ideas[idea]=details

def incNbrIdea(idea):
    details=ideas[idea]

    ideas[idea]=(details[0]+1,details[1])

def addNewPointPerIdea(cpt):

    for idea,XYaxe in axes.items():

        XYaxe=np.append(XYaxe,[ideas[idea][0]])
        print (XYaxe)
        axes[idea]=XYaxe
       #np.append(XYaxe,[ideas[idea][0]],axis=1)

      #  Xaxe=np.array(XYaxe[0])
      #  print(Xaxe)
       # Xaxe.append(np.array(cpt))
        #Yaxe=XYaxe[1]
        #Yaxe.append(ideas[idea][0])
        #axes[idea]=(Xaxe,Yaxe)

def main():
    cpt=0

   # addNewPointPerIdea(cpt)
    drawnow(updateFigure)

    with open('../ProjectFiles/logs/SimulationLog.txt',"r+") as f:
        for line in f:
            line.replace("\n","")
            element=line.split(";")

            decNbrIdea(element[2])
            incNbrIdea(element[3].replace("\n",""))

            cpt+=1
            print(cpt)
            if cpt% 100==0:
                addNewPointPerIdea(cpt)
                drawnow(updateFigure)

initIdeas()
main()


import pandas as pd
import numpy as np
import requests
from scipy.optimize import leastsq
from bs4 import BeautifulSoup

from EnergyIntensityIndicators.get_census_data import Asm
from EnergyIntensityIndicators.get_census_data import Econ_census

class Mfg_prices:
    # def __init__(self):
    #     """
    #     Class for importing and interpolating historical energy prices for
    #     the manufacturing sector.
    #
    #     Historical Manufacturing Energy Consumption Survey (MECS) data are
    #     based on prior work from the Pacific Northwest National Laboratory
    #     (PNNL).
    #     """
    #     # Historical MECS data with missing observations estimated
    #     # ad-hoc by PNNL.
    #     self.mecs_historical_prices = pd.read_csv('mecs_historical_prices.csv')

    @staticmethod
    def import_mecs_historical(file_path):
        """
        Import and format csv of historical fuel prices from Manufacturing
        Energy Consumption Survey (MECS). MECS was conducted every three
        years from 1985 - 1994 and every four years since.

        Parameters
        ----------
        file_path : str
            File path of csv.

        Returns
        -------
        mecs_prices : dataframe

        """

        mecs_prices = pd.read_csv(file_path)

        mecs_prices = pd.melt(mecs_prices, id_vars=['NAICS', 'fuel'],
                              var_name='year', value_name='mecs_price')

        return mecs_prices

    @staticmethod
    def check_recent_mecs(latest_year, last_historical_year):
        """
        MECS has been conducted at four-year intervals since 1994. New data
        will need to be manually downloaded, formatted, and added to the
        historical price file, 'mfg_mecs_energy_prices.csv'.

        Parameters
        ----------
        latest_year : int
            Latest year of historical decomposition.

        last_historical_year : int
            Lastest year of price data available in
            'mfg_mecs_energy_prices.csv'.

        Returns
        -------

        """

        if latest_year not in range(2014, 2040, 4):
            print('{} is not a MECS year.'.format(latest_year))

            return

        if latest_year <= last_historical_year:

            print("Historical MECS prices are latest available")

            return

        else:

            check_url = 'https://www.eia.gov/consumption/manufacturing/' + \
                        'data/{}/xls/table7_2.xlsx'.format(str(latest_year))

            r = requests.get(check_url)

            soup = BeautifulSoup(r.text, 'html.parser')

            # Check if updated MECS are available by requesting Excel file.
            # If no Excel file exists, requests will return the
            # Consumption and Efficiency page. If the Excel file does exist,
            # there will be an exception, which prompts the user to download
            # and format the data.
            try:
                title = soup.title.name

            except AttributeError:
                print("Updated mecs data are now available.\n" +
                      "Please download Tables 3.1, 3.2, 7.2, and 7.6\nand update" +
                      "relevant csv files")

            else:
                print("Updated MECS data are not yet available")

            finally:
                return

    # @staticmethod
    # def import_asm_historical(file_path):
    #     """"
    #     Prices in $/MMBtu
    #     """
    #
    #
    # @staticmethod
    # def get_census_prices(latest_year, start_year=1983):
    #     """
    #     Get fuel prices from Census Bureau's Annual Survey of
    #     Manufacturers and Economic Census (years ending in 2 and 7)
    #
    #     Parameters
    #     ----------
    #     latest_year : int
    #         Most recent year of historical LMDI analysis.
    #
    #     start_year : int, default 1983
    #         Beginning year of price data.
    #
    #     Returns
    #     -------
    #     asm_prices : pandas.Series
    #         Pandas series of ASM price data from YYYY - latest_year
    #     """
    #     year_range = range(start_year, latest_year + 1)
    #
    #     asm_prices = pd.DataFrame()
    #
    #
    #
    #     return asm_prices

    @staticmethod
    def price_func(asm_prices, *params):
        """
        Calculates predicted fuel price in terms of current and lagged fuel
        prices.
        """
        a, b = params

        for index_, price in enumerate(asm_prices):
            if index_ == 0:
                predicted_price_series = np.array([0])
            else:
                predicted_price = a*price + b*asm_prices[index_-1]
                predicted_price_series = np.vstack([predicted_price_series,
                                                    predicted_price])

        early_mecs = predicted_price_series.flatten()[2:12:3]
        later_mecs = predicted_price_series.flatten()[15:len(asm_prices):4]
        final_mecs = np.append(early_mecs, later_mecs)
        # Align with available MECS
        final_mecs = final_mecs[[x[0] for x in enumerate(mecs_prices)]]
        return final_mecs

    @staticmethod
    def residuals(params, asm_prices, mecs_prices, price_func):

        return mecs_prices - price_func(asm_prices, *params)

    @staticmethod
    def calc_predicted_coeffs(asm_prices, mecs_prices, start_params):
        """
        Parameters
        ----------
        asm_prices : numpy.array
            Array of ASM prices

        mecs_prices : numpy.array
            Array of MECS prices, including nan values

        start_params : list
            starting parameters for optimization

        Returns
        -------
        coeff : np.array
            parameters of price prediction equation

        Examples
        --------
        @calmc successully ran this using data from the Gas-311 tab (
        copying cells C6:E42 to clipboard) in Ind_Gas_Prices_123019.xlsx.
        >>>price_data = pd.read_clipboard()
        >>># Using read_clipboard introduces a column of nans
        >>>price_data.dropna(axis=1, how='all', inplace=True)
        >>>price_data.columns = ['year', 'asm_prices', 'mecs_prices']
        >>>price_data.head()
           year  asm_prices  mecs_prices
        0  1983        4.05          NaN
        1  1984        4.10          NaN
        2  1985        3.83         4.21
        3  1986        3.14          NaN
        4  1987        2.85          NaN

        >>>asm_prices = price_data.asm_prices.values
        >>>mecs_prices = price_data.mecs_prices.dropna().values
        >>>coeff = predict_prices(asm_prices, mecs_prices, [1, 1])
        >>>coeff
        array([0.91430597, 0.1074011 ])
        """

        coeff, flag = leastsq(Mfg_prices.residuals, start_params,
                              args=(asm_prices, mecs_prices,
                                    Mfg_prices.price_func))

        return coeff

    @staticmethod
    def calc_predicted_prices(coeff, asm_prices):
        """Calculate predicted price with leastsq coeffs"""
        return Mfg_prices.price_func(asm_prices, *coeff)

    @staticmethod
    def resid_filler(price_df):
        """
        Apply to a dataframe that includes year and residual.

        price_df.columns = ['year', 'asm_price', 'MECS_price', 'predicted',
        'residual']

        Parameters
        ----------
        residual : np.array
            Array of residuals, including nan values.
        """

        increment_years = price_df.dropna()['year'].values

        # Lines 165 - 173 should be placed in separate method.
        predicted_prices = Mfg_prices.calc_predicted_prices(coeff, asm_prices)
        predicted_prices = pd.DataFrame(predicted_prices,
                                        columns=['predicted'],
                                        index=increment_years)

        price_df.set_index('year', inplace=True)
        price_df = pd.concat([price_df, predicted_prices], axis=1)
        price_df['residual'] = price_df.MECS_price - price_df.predicted

        price_df['interp_resid'] = np.nan

        for index, y_ in enumerate(increment_years):
            if index > 0:
                year_before = increment_years[index - 1]
                num_years = y_ - year_before
                resid_year_before = price_df.xs(year_before)['residual']
                resid_y_ = price_df.xs(y_)['residual']
                increment = 1 / num_years
                for delta in range(num_years):
                    value = resid_year_before * (1 - increment * delta) + \
                        resid_y_ * (increment * delta)
                    year = year_before + delta
                    price_df.loc[year, 'interp_resid'] = value

        # Fill in remaining missing values
        price_df.loc[increment_years[-1], 'interp_resid'] = \
            price_df.xs(increment_years[-1])['residual']
        price_df.interp_resid.fillna(method='ffill', inplace=True)

        price_df['calibrated_prediction'] = \
            price_df.predicted + price_df.interp_resid

        price_df.reset_index(inplace=True)

        return price_df

    def calc_calibrated_predicted_price(self, latest_year, fuel_type, naics):
        """
        Return the calibrated prediced prices, which is calculated as the

        """
        # Get asm prices from separate method?

        mecs_prices = self.mecs_historical_prices(fuel_type, naics)

        fit_coeffs = Mfg_prices.calc_predicted_coeffs()
        predicted = Mfg_prices.calc_predicted_prices()

        price_df['calibrated_prediction'] = \
            price_df.interp_resid + price_df.predicted

        return price_df

    # @staticmethod
    # def interpolate_residuals(predicted_prices, price_df):
    #     """Interpolate residuals"""
    #
    #     price_df_updated = price_df.copy(deep=True)
    #     price_df_updated['predicted_price'] = predicted_prices
    #     price_df_updated['residual'] = price_df_updated.mecs.subtract(
    #         price_df_updated.predicted_price
    #         )
    #
    #     fill = price_df_updated.dropna(subset=['residual'], axis=0).year.diff()
    #     # fill.fillna(0, inplace=True)
    #
    #     price_df_updated['fill'] = fill
    #
    #
    #     return interpolated_resid
    #

    # Predicted prices are use where? Check documentation to figure out [Search NAICS]


# -*- coding: utf-8 -*-
from openerp import http
from openerp.http import request
from openerp.addons.base.ir.ir_qweb import QWeb

class QWebExtend(QWeb):

    def render_tag_call_assets(self, element, template_attributes, generated_attributes, qwebcontext):
        # do notcheck whether we want to return minified assets
        cr = qwebcontext.get('request') and qwebcontext['request'].cr or None
        uid = qwebcontext.get('request') and qwebcontext['request'].uid or None
        do_not_minify = self.user_has_groups(cr, uid, groups='base.group_user') if cr and uid else False
        if do_not_minify:
            # do not returned minified assets
            # by pretending that the dbug mode is active
            qwebcontext['debug'] = True
        return super(QWebExtend, self).render_tag_call_assets(element, template_attributes, generated_attributes, qwebcontext)


"""Podnet Library for Raspberry Pi"""

import json
import time
from RF24 import *
from RF24Network import *

octlit = lambda n: int(n, 8)
POD_ADDRESS = octlit("00")
DEFAULT_STRING_ENCODING = "utf-8"

class UnconfiguredNodeException(Exception):
    pass

def chunkstring(string, length):
    return (string[0 + i : length + i] for i in range(0, len(string), length))

class Podnet:
    def __init__(self, thing_id, debug=False):
        """

        :param str thing_id: Thing ID generated on the dashboard.
        :param bool debug: Set this to True to get detailed information on connection
        """
        radio = RF24(RPI_V2_GPIO_P1_15, RPI_V2_GPIO_P1_24, BCM2835_SPI_SPEED_8MHZ)
        network = RF24Network(radio)

        self.thing_id = thing_id
        self.radio = radio
        self.network = network
        self.node_id = octlit("01")

        radio.begin()
        time.sleep(0.1)
        network.begin(90, self.node_id)
        if debug:
            radio.printDetails()

        # Asking Pod for next available address
        if debug:
            print("Sending msg to Pod for new Node ID assignment.")
        network.update()
        msg = json.dumps({"tid": self.thing_id, "m": "assignNodeID"})

        if self.send_with_retry_to_pod(msg):
            print("Assign NodeID msg sent")
        else:
            raise UnconfiguredNodeException("Unable to send msg for assigning Node ID.")

        # network.write(RF24NetworkHeader(POD_ADDRESS), msg)
        time.sleep(1)

        # Listen for response from Pod
        if debug:
            print("Waiting for response from Pod for Node ID")

        if debug:
            print("Listening for NodeID response...")
        while True:
            network.update()
            if network.available():
                if debug:
                    print("Received a response.")
                try:
                    header, payload = network.read(100)
                    msg = payload.decode(DEFAULT_STRING_ENCODING)
                    resp = json.loads(msg)
                    self.node_id = octlit(str(resp["response"]))
                    if debug:
                        print("Changing Node ID...")
                    network.begin(90, self.node_id)
                    break
                except json.decoder.JSONDecodeError:
                    print("Unable to decode JSON msg.")
                    raise UnconfiguredNodeException(
                        "Unable to configure node because response from Pod could not be parsed."
                    )
            time.sleep(1)

        # Case when new address wasn't assigned
        if self.node_id == octlit("01"):
            raise UnconfiguredNodeException(
                "This node is unconfigured. No Node ID was assigned by Pod."
            )

        print("Node configured.")

    def send_with_retry_to_pod(self, msg, msg_type=0, retry=5):
        """
        Sends a message to Pod, if it fails, retry given no. of times
        :param str msg: Message to send to the Pod
        :param int msg_type: A number between 1-127 for specifying header type in the message.
        :param int retry: Number of times to retry if the message does not goes through.
        :return: Whether the message was successfully sent to Pod?
        :rtype: bool
        """
        if isinstance(msg, str):
            msg = msg.encode(DEFAULT_STRING_ENCODING)

        attempts = 0
        while not self.network.write(RF24NetworkHeader(octlit("00"), msg_type), msg):
            if attempts > retry:
                return False
            attempts += 1
            time.sleep(1)

        return True

    def send_multipart_message(self, msg, debug=False):
        """
        Send a message longer than 144 bytes, by dividing it in multiple pieces

        :param str msg: Message to send
        :param bool debug: Set this to True to get detailed information while every part of message is being sent.
        :return: Whether the message was successfully sent?
        :rtype: bool
        """
        parts = list(chunkstring(msg, 72))
        part_sent = list()

        # Msg has only one part
        if len(parts) == 1:
            return self.send_with_retry_to_pod(parts[0])

        # Msg has multiple parts
        else:
            i = 0
            while i < len(parts):
                # Send last msg with header.type = 80
                if i == (len(parts) - 1):
                    part_sent.append(self.send_with_retry_to_pod(parts[i], 80))

                # All other msgs have header.type = 79
                else:
                    part_sent.append(self.send_with_retry_to_pod(parts[i], 79))

                if debug:
                    print(str(parts[i]) + " : " + str(part_sent[i]))

                i += 1
                time.sleep(1)

        return all(part_sent)

    def send_to_cloud(self, msg, debug=False):
        """
        Send message directly to the your dashboard on https://dashboard.thepodnet.com
        :param str msg: Message to send
        :param bool debug: Set this to True to get detailed information while message is being sent to the cloud.
        :return: Whether the message was successfully sent?
        :rtype: bool
        """
        data = json.dumps({"tid": self.thing_id, "p": msg, "m": "sendToCloud"})
        return self.send_multipart_message(data, debug=debug)

    def send_to(self, msg, other_thing_id, debug=True):
        """
        Send message directly to other node that is connected to the Pod.
        :param str msg: Message to send
        :param str other_thing_id: Thing ID of the other device to which the message has to be sent.
        :param bool debug: Set this to True to get detailed information while the message is being sent to the other device.
        :return: Whether the message was successfully sent?
        :rtype: bool
        """
        data = json.dumps({"tid": self.thing_id, "p": msg, "m": "sendTo", "otid": other_thing_id})
        return self.send_multipart_message(data, debug=debug)

    def recv(self):
        """
        Receive messages sent from the cloud(https://dashboard.thepodnet.com) or any other node.
        :return: Message that was sent from the cloud or any other node.
        :rtype: str
        """
        while True:
            self.network.update()
            if self.network.available():
                header, payload = self.network.read(144)
                return payload.decode(DEFAULT_STRING_ENCODING)

            time.sleep(0.2)


# This file is part of Radicale Server - Calendar Server
# Copyright © 2008 Nicolas Kandel
# Copyright © 2008 Pascal Halter
# Copyright © 2014 Jean-Marc Martins
# Copyright © 2008-2017 Guillaume Ayoub
# Copyright © 2017-2018 Unrud <unrud@outlook.com>
#
# This library is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
import math
import sys
from hashlib import md5
from random import getrandbits

import vobject

from radicale import pathutils
from radicale.item import filter as radicale_filter

def predict_tag_of_parent_collection(vobject_items):
    if len(vobject_items) != 1:
        return ""
    if vobject_items[0].name == "VCALENDAR":
        return "VCALENDAR"
    if vobject_items[0].name in ("VCARD", "VLIST"):
        return "VADDRESSBOOK"
    return ""

def predict_tag_of_whole_collection(vobject_items, fallback_tag=None):
    if vobject_items and vobject_items[0].name == "VCALENDAR":
        return "VCALENDAR"
    if vobject_items and vobject_items[0].name in ("VCARD", "VLIST"):
        return "VADDRESSBOOK"
    if not fallback_tag and not vobject_items:
        # Maybe an empty address book
        return "VADDRESSBOOK"
    return fallback_tag

def check_and_sanitize_items(vobject_items, is_collection=False, tag=None):
    """Check vobject items for common errors and add missing UIDs.

    ``is_collection`` indicates that vobject_item contains unrelated
    components.

    The ``tag`` of the collection.

    """
    if tag and tag not in ("VCALENDAR", "VADDRESSBOOK"):
        raise j.exceptions.Value("Unsupported collection tag: %r" % tag)
    if not is_collection and len(vobject_items) != 1:
        raise j.exceptions.Value("Item contains %d components" % len(vobject_items))
    if tag == "VCALENDAR":
        if len(vobject_items) > 1:
            raise j.exceptions.Base("VCALENDAR collection contains %d " "components" % len(vobject_items))
        vobject_item = vobject_items[0]
        if vobject_item.name != "VCALENDAR":
            raise j.exceptions.Value("Item type %r not supported in %r " "collection" % (vobject_item.name, tag))
        component_uids = set()
        for component in vobject_item.components():
            if component.name in ("VTODO", "VEVENT", "VJOURNAL"):
                component_uid = get_uid(component)
                if component_uid:
                    component_uids.add(component_uid)
        component_name = None
        object_uid = None
        object_uid_set = False
        for component in vobject_item.components():
            # https://tools.ietf.org/html/rfc4791#section-4.1
            if component.name == "VTIMEZONE":
                continue
            if component_name is None or is_collection:
                component_name = component.name
            elif component_name != component.name:
                raise j.exceptions.Value(
                    "Multiple component types in object: %r, %r" % (component_name, component.name)
                )
            if component_name not in ("VTODO", "VEVENT", "VJOURNAL"):
                continue
            component_uid = get_uid(component)
            if not object_uid_set or is_collection:
                object_uid_set = True
                object_uid = component_uid
                if not component_uid:
                    if not is_collection:
                        raise j.exceptions.Value("%s component without UID in object" % component_name)
                    component_uid = find_available_uid(component_uids.__contains__)
                    component_uids.add(component_uid)
                    if hasattr(component, "uid"):
                        component.uid.value = component_uid
                    else:
                        component.add("UID").value = component_uid
            elif not object_uid or not component_uid:
                raise j.exceptions.Value("Multiple %s components without UID in " "object" % component_name)
            elif object_uid != component_uid:
                raise j.exceptions.Value(
                    "Multiple %s components with different UIDs in object: "
                    "%r, %r" % (component_name, object_uid, component_uid)
                )
            # vobject interprets recurrence rules on demand
            try:
                component.rruleset
            except Exception as e:
                raise j.exceptions.Value("invalid recurrence rules in %s" % component.name) from e
    elif tag == "VADDRESSBOOK":
        # https://tools.ietf.org/html/rfc6352#section-5.1
        object_uids = set()
        for vobject_item in vobject_items:
            if vobject_item.name == "VCARD":
                object_uid = get_uid(vobject_item)
                if object_uid:
                    object_uids.add(object_uid)
        for vobject_item in vobject_items:
            if vobject_item.name == "VLIST":
                # Custom format used by SOGo Connector to store lists of
                # contacts
                continue
            if vobject_item.name != "VCARD":
                raise j.exceptions.Value("Item type %r not supported in %r " "collection" % (vobject_item.name, tag))
            object_uid = get_uid(vobject_item)
            if not object_uid:
                if not is_collection:
                    raise j.exceptions.Value("%s object without UID" % vobject_item.name)
                object_uid = find_available_uid(object_uids.__contains__)
                object_uids.add(object_uid)
                if hasattr(vobject_item, "uid"):
                    vobject_item.uid.value = object_uid
                else:
                    vobject_item.add("UID").value = object_uid
    else:
        for i in vobject_items:
            raise j.exceptions.Value(
                "Item type %r not supported in %s collection" % (i.name, repr(tag) if tag else "generic")
            )

def check_and_sanitize_props(props):
    """Check collection properties for common errors."""
    tag = props.get("tag")
    if tag and tag not in ("VCALENDAR", "VADDRESSBOOK"):
        raise j.exceptions.Value("Unsupported collection tag: %r" % tag)

def find_available_uid(exists_fn, suffix=""):
    """Generate a pseudo-random UID"""
    # Prevent infinite loop
    for _ in range(1000):
        r = "%016x" % getrandbits(128)
        name = "%s-%s-%s-%s-%s%s" % (r[:8], r[8:12], r[12:16], r[16:20], r[20:], suffix)
        if not exists_fn(name):
            return name
    # something is wrong with the PRNG
    raise j.exceptions.Base("No unique random sequence found")

def get_etag(text):
    """Etag from collection or item.

    Encoded as quoted-string (see RFC 2616).

    """
    etag = md5()
    etag.update(text.encode("utf-8"))
    return '"%s"' % etag.hexdigest()

def get_uid(vobject_component):
    """UID value of an item if defined."""
    return vobject_component.uid.value if hasattr(vobject_component, "uid") else None

def get_uid_from_object(vobject_item):
    """UID value of an calendar/addressbook object."""
    if vobject_item.name == "VCALENDAR":
        if hasattr(vobject_item, "vevent"):
            return get_uid(vobject_item.vevent)
        if hasattr(vobject_item, "vjournal"):
            return get_uid(vobject_item.vjournal)
        if hasattr(vobject_item, "vtodo"):
            return get_uid(vobject_item.vtodo)
    elif vobject_item.name == "VCARD":
        return get_uid(vobject_item)
    return None

def find_tag(vobject_item):
    """Find component name from ``vobject_item``."""
    if vobject_item.name == "VCALENDAR":
        for component in vobject_item.components():
            if component.name != "VTIMEZONE":
                return component.name or ""
    return ""

def find_tag_and_time_range(vobject_item):
    """Find component name and enclosing time range from ``vobject item``.

    Returns a tuple (``tag``, ``start``, ``end``) where ``tag`` is a string
    and ``start`` and ``end`` are POSIX timestamps (as int).

    This is intened to be used for matching against simplified prefilters.

    """
    tag = find_tag(vobject_item)
    if not tag:
        return (tag, radicale_filter.TIMESTAMP_MIN, radicale_filter.TIMESTAMP_MAX)
    start = end = None

    def range_fn(range_start, range_end, is_recurrence):
        nonlocal start, end
        if start is None or range_start < start:
            start = range_start
        if end is None or end < range_end:
            end = range_end
        return False

    def infinity_fn(range_start):
        nonlocal start, end
        if start is None or range_start < start:
            start = range_start
        end = radicale_filter.DATETIME_MAX
        return True

    radicale_filter.visit_time_ranges(vobject_item, tag, range_fn, infinity_fn)
    if start is None:
        start = radicale_filter.DATETIME_MIN
    if end is None:
        end = radicale_filter.DATETIME_MAX
    try:
        return tag, math.floor(start.timestamp()), math.ceil(end.timestamp())
    except ValueError as e:
        if str(e) == ("offset must be a timedelta representing a whole " "number of minutes") and sys.version_info < (
            3,
            6,
        ):
            raise j.exceptions.Base("Unsupported in Python < 3.6: %s" % e) from e
        raise

class Item:
    def __init__(
        self,
        collection_path=None,
        collection=None,
        vobject_item=None,
        href=None,
        last_modified=None,
        text=None,
        etag=None,
        uid=None,
        name=None,
        component_name=None,
        time_range=None,
    ):
        """Initialize an item.

        ``collection_path`` the path of the parent collection (optional if
        ``collection`` is set).

        ``collection`` the parent collection (optional).

        ``href`` the href of the item.

        ``last_modified`` the HTTP-datetime of when the item was modified.

        ``text`` the text representation of the item (optional if
        ``vobject_item`` is set).

        ``vobject_item`` the vobject item (optional if ``text`` is set).

        ``etag`` the etag of the item (optional). See ``get_etag``.

        ``uid`` the UID of the object (optional). See ``get_uid_from_object``.

        ``name`` the name of the item (optional). See ``vobject_item.name``.

        ``component_name`` the name of the primary component (optional).
        See ``find_tag``.

        ``time_range`` the enclosing time range.
        See ``find_tag_and_time_range``.

        """
        if text is None and vobject_item is None:
            raise j.exceptions.Value("at least one of 'text' or 'vobject_item' must be set")
        if collection_path is None:
            if collection is None:
                raise j.exceptions.Value("at least one of 'collection_path' or " "'collection' must be set")
            collection_path = collection.path
        assert collection_path == pathutils.strip_path(pathutils.sanitize_path(collection_path))
        self._collection_path = collection_path
        self.collection = collection
        self.href = href
        self.last_modified = last_modified
        self._text = text
        self._vobject_item = vobject_item
        self._etag = etag
        self._uid = uid
        self._name = name
        self._component_name = component_name
        self._time_range = time_range

    def serialize(self):
        if self._text is None:
            try:
                self._text = self.vobject_item.serialize()
            except Exception as e:
                raise j.exceptions.Base(
                    "Failed to serialize item %r from %r: %s" % (self.href, self._collection_path, e)
                ) from e
        return self._text

    @property
    def vobject_item(self):
        if self._vobject_item is None:
            try:
                self._vobject_item = vobject.readOne(self._text)
            except Exception as e:
                raise j.exceptions.Base(
                    "Failed to parse item %r from %r: %s" % (self.href, self._collection_path, e)
                ) from e
        return self._vobject_item

    @property
    def etag(self):
        """Encoded as quoted-string (see RFC 2616)."""
        if self._etag is None:
            self._etag = get_etag(self.serialize())
        return self._etag

    @property
    def uid(self):
        if self._uid is None:
            self._uid = get_uid_from_object(self.vobject_item)
        return self._uid

    @property
    def name(self):
        if self._name is None:
            self._name = self.vobject_item.name or ""
        return self._name

    @property
    def component_name(self):
        if self._component_name is not None:
            return self._component_name
        return find_tag(self.vobject_item)

    @property
    def time_range(self):
        if self._time_range is None:
            self._component_name, *self._time_range = find_tag_and_time_range(self.vobject_item)
        return self._time_range

    def prepare(self):
        """Fill cache with values."""
        orig_vobject_item = self._vobject_item
        self.serialize()
        self.etag
        self.uid
        self.name
        self.time_range
        self.component_name
        self._vobject_item = orig_vobject_item


"""Returns tidy3d simulation from gdsfactory Component."""
import warnings
from typing import Dict, Optional

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import pydantic
import tidy3d as td

import gdsfactory as gf
from gdsfactory.component import Component
from gdsfactory.components.extension import move_polar_rad_copy
from gdsfactory.config import logger
from gdsfactory.routing.sort_ports import sort_ports_x, sort_ports_y
from gdsfactory.simulation.gtidy3d.materials import (
    MATERIAL_NAME_TO_TIDY3D_INDEX,
    MATERIAL_NAME_TO_TIDY3D_NAME,
    get_index,
    get_medium,
)
from gdsfactory.tech import LAYER_STACK, LayerStack
from gdsfactory.types import ComponentOrFactory, Float2

@pydantic.validate_arguments
def get_simulation(
    component: ComponentOrFactory,
    port_extension: Optional[float] = 4.0,
    layer_stack: LayerStack = LAYER_STACK,
    thickness_pml: float = 1.0,
    xmargin: float = 0,
    ymargin: float = 0,
    xmargin_left: float = 0,
    xmargin_right: float = 0,
    ymargin_top: float = 0,
    ymargin_bot: float = 0,
    zmargin: float = 1.0,
    clad_material: str = "sio2",
    port_source_name: str = "o1",
    port_margin: float = 0.5,
    port_source_offset: float = 0.1,
    distance_source_to_monitors: float = 0.2,
    resolution: float = 50,
    wavelength_start: float = 1.50,
    wavelength_stop: float = 1.60,
    wavelength_points: int = 50,
    plot_modes: bool = False,
    num_modes: int = 2,
    run_time_ps: float = 10.0,
    dispersive: bool = False,
    material_name_to_tidy3d_index: Dict[str, float] = MATERIAL_NAME_TO_TIDY3D_INDEX,
    material_name_to_tidy3d_name: Dict[str, str] = MATERIAL_NAME_TO_TIDY3D_NAME,
    is_3d: bool = True,
    with_all_monitors: bool = False,
) -> td.Simulation:
    r"""Returns Simulation object from gdsfactory.component

    based on GDS example
    https://simulation.cloud/docs/html/examples/ParameterScan.html

    .. code::

         top view
              ________________________________
             |                               |
             | xmargin_left                  | port_extension
             |<------>          port_margin ||<-->
          ___|___________          _________||___
             |           \        /          |
             |            \      /           |
             |             ======            |
             |            /      \           |
          ___|___________/        \__________|___
             |   |                 <-------->|
             |   |ymargin_bot   xmargin_right|
             |   |                           |
             |___|___________________________|

        side view
              ________________________________
             |                     |         |
             |                     |         |
             |                   zmargin_top |
             |xmargin_left         |         |
             |<---> _____         _|___      |
             |     |     |       |     |     |
             |     |     |       |     |     |
             |     |_____|       |_____|     |
             |       |                       |
             |       |                       |
             |       |zmargin_bot            |
             |       |                       |
             |_______|_______________________|

    Args:
        component: gdsfactory Component.
        port_extension: extend ports beyond the PML.
        layer_stack: contains layer numbers (int, int) to thickness, zmin.
        thickness_pml: PML thickness (um).
        xmargin: left/right distance from component to PML.
        xmargin_left: left distance from component to PML.
        xmargin_right: right distance from component to PML.
        ymargin: left/right distance from component to PML.
        ymargin_top: top distance from component to PML.
        ymargin_bot: bottom distance from component to PML.
        zmargin: thickness for cladding above and below core.
        clad_material: material for cladding.
        port_source_name: input port name.
        port_margin: margin on each side of the port.
        distance_source_to_monitors: in (um) source goes before monitors.
        port_source_offset: mode solver workaround.
            positive moves source forward, negative moves source backward.
        resolution: in pixels/um (20: for coarse, 120: for fine)
        wavelength_start: in (um).
        wavelength_stop: in (um).
        wavelength_points: number of wavelengths.
        plot_modes: plot source modes.
        num_modes: number of modes to plot.
        run_time_ps: make sure it's sufficient for the fields to decay.
            defaults to 10ps and counts on automatic shutoff to stop earlier if needed.
        dispersive: False uses constant refractive index materials.
            True adds wavelength depending materials.
            Dispersive materials require more computation.
        material_name_to_tidy3d_index: not dispersive materials have a constant index.
        material_name_to_tidy3d_name: dispersive materials have a wavelength
            dependent index. Maps layer_stack names with tidy3d material database names.
        is_3d: if False, does not consider Z dimension for faster simulations.
        with_all_monitors: if True, includes field monitors which increase results file size.

    .. code::

        import matplotlib.pyplot as plt
        import gdsfactory as gf
        import gdsfactory.simulation.tidy3d as gt

        c = gf.components.bend_circular()
        sim = gt.get_simulation(c)
        gt.plot_simulation(sim)

    """
    component = component() if callable(component) else component
    assert isinstance(component, Component)

    layer_to_thickness = layer_stack.get_layer_to_thickness()
    layer_to_material = layer_stack.get_layer_to_material()
    layer_to_zmin = layer_stack.get_layer_to_zmin()
    # layer_to_sidewall_angle = layer_stack.get_layer_to_sidewall_angle()

    if dispersive:
        material_name_to_tidy3d = material_name_to_tidy3d_name
    else:
        material_name_to_tidy3d = material_name_to_tidy3d_index

    assert isinstance(
        component, Component
    ), f"component needs to be a gf.Component, got Type {type(component)}"
    if port_source_name not in component.ports:
        warnings.warn(
            f"port_source_name={port_source_name} not in {component.ports.keys()}"
        )
        port_source = component.get_ports_list(port_type="optical")[0]
        port_source_name = port_source.name
        warnings.warn(f"Selecting port_source_name={port_source_name} instead.")

    component_padding = gf.add_padding_container(
        component,
        default=0,
        top=ymargin or ymargin_top,
        bottom=ymargin or ymargin_bot,
        left=xmargin or xmargin_left,
        right=xmargin or xmargin_right,
    )
    component_extended = (
        gf.components.extension.extend_ports(
            component=component_padding, length=port_extension, centered=True
        )
        if port_extension
        else component_padding
    )

    gf.show(component_extended)
    component_extended = component_extended.flatten()

    component_ref = component_padding.ref()
    component_ref.x = 0
    component_ref.y = 0

    clad_material_name_or_index = material_name_to_tidy3d[clad_material]
    clad = td.Structure(
        geometry=td.Box(
            size=(td.inf, td.inf, td.inf),
            center=(0, 0, 0),
        ),
        medium=get_medium(name_or_index=clad_material_name_or_index),
    )
    structures = [clad]

    layers_thickness = [
        layer_to_thickness[layer]
        for layer in component.get_layers()
        if layer in layer_to_thickness
    ]

    if len(layer_to_thickness) < 1:
        raise ValueError(f"{component.get_layers()} not in {layer_to_thickness.keys()}")

    t_core = max(layers_thickness)
    cell_thickness = (
        thickness_pml + t_core + thickness_pml + 2 * zmargin
        if is_3d
        else 1 / resolution
    )

    sim_size = [
        component_ref.xsize + 2 * thickness_pml,
        component_ref.ysize + 2 * thickness_pml,
        cell_thickness,
    ]

    for layer in component.layers:
        if layer in layer_to_thickness and layer in layer_to_material:
            thickness = layer_to_thickness[layer]
            zmin = layer_to_zmin[layer] if is_3d else -td.inf
            zmax = zmin + thickness if is_3d else td.inf

            if (
                layer in layer_to_material
                and layer_to_material[layer] in material_name_to_tidy3d
            ):
                name_or_index = material_name_to_tidy3d[layer_to_material[layer]]
                medium = get_medium(name_or_index=name_or_index)
                index = get_index(name_or_index=name_or_index)
                logger.debug(
                    f"Add {layer}, {name_or_index!r}, index = {index:.3f}, "
                    f"thickness = {thickness}, zmin = {zmin}, zmax = {zmax}"
                )

                polygons = td.PolySlab.from_gds(
                    gds_cell=component_extended,
                    gds_layer=layer[0],
                    gds_dtype=layer[1],
                    axis=2,
                    slab_bounds=(zmin, zmax),
                )

                for polygon in polygons:
                    geometry = td.Structure(
                        geometry=polygon,
                        medium=medium,
                    )
                    structures.append(geometry)
            elif layer not in layer_to_material:
                logger.debug(f"Layer {layer} not in {layer_to_material.keys()}")
            elif layer_to_material[layer] not in material_name_to_tidy3d:
                materials = list(material_name_to_tidy3d.keys())
                logger.debug(f"material {layer_to_material[layer]} not in {materials}")

    # Add source
    port = component_ref.ports[port_source_name]
    angle = port.orientation
    width = port.width + 2 * port_margin
    size_x = width * abs(np.sin(angle * np.pi / 180))
    size_y = width * abs(np.cos(angle * np.pi / 180))
    size_x = 0 if size_x < 0.001 else size_x
    size_y = 0 if size_y < 0.001 else size_y
    size_z = cell_thickness - 2 * zmargin if is_3d else td.inf

    source_size = [size_x, size_y, size_z]
    source_center = port.center.tolist() + [0]  # (x, y, z=0)

    xy_shifted = move_polar_rad_copy(
        np.array(port.center), angle=angle * np.pi / 180, length=port_source_offset
    )
    source_center_offset = xy_shifted.tolist() + [0]  # (x, y, z=0)

    wavelengths = np.linspace(wavelength_start, wavelength_stop, wavelength_points)
    freqs = td.constants.C_0 / wavelengths
    freq0 = td.constants.C_0 / np.mean(wavelengths)
    fwidth = freq0 / 10

    msource = td.ModeSource(
        size=source_size,
        center=source_center,
        source_time=td.GaussianPulse(freq0=freq0, fwidth=fwidth),
        direction="+",
    )

    # Add port monitors
    monitors = {}
    ports = sort_ports_x(sort_ports_y(component_ref.get_ports_list()))
    for port in ports:
        port_name = port.name
        angle = port.orientation
        width = port.width + 2 * port_margin
        size_x = width * abs(np.sin(angle * np.pi / 180))
        size_y = width * abs(np.cos(angle * np.pi / 180))
        size_x = 0 if size_x < 0.001 else size_x
        size_y = 0 if size_y < 0.001 else size_y
        size = (size_x, size_y, size_z)

        # if monitor has a source move monitor inwards
        length = -distance_source_to_monitors if port_name == port_source_name else 0
        xy_shifted = move_polar_rad_copy(
            np.array(port.center), angle=angle * np.pi / 180, length=length
        )
        center = xy_shifted.tolist() + [0]  # (x, y, z=0)

        monitors[port_name] = td.ModeMonitor(
            center=center,
            size=size,
            freqs=freqs,
            mode_spec=td.ModeSpec(num_modes=1),
            name=port.name,
        )

    zcenter = (zmax + zmin) / 2 if is_3d else 0
    domain_monitor = td.FieldMonitor(
        center=[0, 0, zcenter],
        size=[sim_size[0], sim_size[1], 0] if is_3d else [td.inf, td.inf, 0],
        freqs=[freq0],
        name="field",
    )
    monitors = list(monitors.values())
    monitors += [domain_monitor] if with_all_monitors else []

    sim = td.Simulation(
        size=sim_size,
        grid_size=3 * [1 / resolution],
        structures=structures,
        sources=[msource],
        monitors=monitors,
        run_time=20 * run_time_ps / fwidth,
        pml_layers=3 * [td.PML()] if is_3d else [td.PML(), td.PML(), None],
    )

    if plot_modes:
        src_plane = td.Box(center=source_center_offset, size=source_size)
        ms = td.plugins.ModeSolver(simulation=sim, plane=src_plane, freq=freq0)
        mode_spec = td.ModeSpec(num_modes=num_modes)
        modes = ms.solve(mode_spec=mode_spec)

        print(
            "Effective index of computed modes: ",
            ", ".join([f"{mode.n_eff:1.4f}" for mode in modes]),
        )

        if is_3d:
            fig, axs = plt.subplots(num_modes, 2, figsize=(12, 12))
        else:
            fig, axs = plt.subplots(num_modes, 3, figsize=(12, 12))

        for mode_ind in range(num_modes):
            if is_3d:
                abs(modes[mode_ind].field_data.Ey).plot(
                    x="y", y="z", cmap="magma", ax=axs[mode_ind, 0]
                )
                abs(modes[mode_ind].field_data.Ez).plot(
                    x="y", y="z", cmap="magma", ax=axs[mode_ind, 1]
                )
            else:
                abs(modes[mode_ind].field_data.Ex).plot(ax=axs[mode_ind, 0])
                abs(modes[mode_ind].field_data.Ey).plot(ax=axs[mode_ind, 1])
                abs(modes[mode_ind].field_data.Ez).plot(ax=axs[mode_ind, 2])

                axs[mode_ind, 0].set_title(f"|Ex|: mode_index={mode_ind}")
                axs[mode_ind, 1].set_title(f"|Ey|: mode_index={mode_ind}")
                axs[mode_ind, 2].set_title(f"|Ez|: mode_index={mode_ind}")

        if is_3d:
            axs[mode_ind, 0].set_aspect("equal")
            axs[mode_ind, 1].set_aspect("equal")
        plt.show()
    return sim

def plot_simulation_yz(
    sim: td.Simulation,
    z: float = 0.0,
    y: float = 0.0,
    wavelength: Optional[float] = 1.55,
    figsize: Float2 = (11, 4),
):
    """Returns Simulation visual representation.
    returns two views for 3D component and one view for 2D

    Args:
        sim: simulation object
        z: (um)
        y: (um)
        wavelength: (um) for epsilon plot if None plot structures.
        figsize: figure size
    """
    fig = plt.figure(figsize=figsize)
    if sim.size[2] > 0.1 and sim.size[1] > 0.1:
        gs = mpl.gridspec.GridSpec(1, 2, figure=fig, width_ratios=[1, 1.4])
        ax1 = fig.add_subplot(gs[0, 0])
        ax2 = fig.add_subplot(gs[0, 1])
        if wavelength:
            freq = td.constants.C_0 / wavelength
            sim.plot_eps(z=z, ax=ax1, freq=freq)
            sim.plot_eps(y=y, ax=ax2, freq=freq)
        else:
            sim.plot(z=z, ax=ax1)
            sim.plot(y=y, ax=ax2)
    elif sim.size[2] > 0.1:  # 2D grating sim_size_y = 0
        gs = mpl.gridspec.GridSpec(1, 1, figure=fig, width_ratios=[1])
        ax1 = fig.add_subplot(gs[0, 0])
        if wavelength:
            freq = td.constants.C_0 / wavelength
            sim.plot_eps(y=y, ax=ax1, freq=freq)
        else:
            sim.plot(y=y, ax=ax1)

    else:  # 2D planar component size_z = 0
        gs = mpl.gridspec.GridSpec(1, 1, figure=fig, width_ratios=[1])
        ax1 = fig.add_subplot(gs[0, 0])
        if wavelength:
            freq = td.constants.C_0 / wavelength
            sim.plot_eps(z=z, ax=ax1, freq=freq)
        else:
            sim.plot(z=z, ax=ax1)

    plt.show()
    return fig

def plot_simulation_xz(
    sim: td.Simulation,
    x: float = 0.0,
    z: float = 0.0,
    wavelength: Optional[float] = 1.55,
    figsize: Float2 = (11, 4),
):
    """Returns figure with two axis of the Simulation.

    Args:
        sim: simulation object
        x: (um)
        z: (um)
        wavelength: (um) for epsilon plot if None plot structures.
        figsize: figure size
    """
    fig = plt.figure(figsize=figsize)
    gs = mpl.gridspec.GridSpec(1, 2, figure=fig, width_ratios=[1, 1.4])
    ax1 = fig.add_subplot(gs[0, 0])
    ax2 = fig.add_subplot(gs[0, 1])
    if wavelength:
        freq = td.constants.C_0 / wavelength
        sim.plot_eps(z=z, ax=ax1, freq=freq)
        sim.plot_eps(x=x, ax=ax2, freq=freq)
    else:
        sim.plot(z=z, ax=ax1)
        sim.plot(x=x, ax=ax2)
    plt.show()
    return fig

plot_simulation = plot_simulation_yz

if __name__ == "__main__":
    # c = gf.components.mmi1x2()
    # c = gf.components.bend_circular(radius=2)
    # c = gf.components.crossing()
    # c = gf.c.straight_rib()

    c = gf.c.straight(length=3)
    sim = get_simulation(c, plot_modes=False, is_3d=False)

    plot_simulation(sim)

    # filepath = pathlib.Path(__file__).parent / "extra" / "wg2d.json"
    # filepath.write_text(sim.json())

    # sim.plotly(z=0)
    # plot_simulation_yz(sim, wavelength=1.55)
    # fig = plt.figure(figsize=(11, 4))
    # gs = mpl.gridspec.GridSpec(1, 2, figure=fig, width_ratios=[1, 1.4])
    # ax1 = fig.add_subplot(gs[0, 0])
    # ax2 = fig.add_subplot(gs[0, 1])
    # sim.plot(z=0.0, ax=ax1)
    # sim.plot(x=0.0, ax=ax2)
    # plt.show()