from collections import deque
from typing import Tuple, Union

from pycocotb.agents.base import AgentWitReset, NOP
from pycocotb.constants import CLK_PERIOD
from pycocotb.hdlSimulator import HdlSimulator
from pycocotb.simCalendar import DONE
from pycocotb.triggers import Timer, WaitWriteOnly, WaitCombRead, Edge

class TristateSignal():
    """
    Container of signals for controll of tristate block

    :ivar ~.i: input - slave to master
    :ivar ~.o: output - master to slave
    :ivar ~.t: master to slave, if 1 the value of o is set to i
    """
    def __init__(self, i: "RtlSignal", o: "RtlSignal", t: "RtlSignal"):
        self.i = i
        self.o = o
        self.t = t

class TristateAgent(AgentWitReset):
    """
    :ivar ~.selfSynchronization: if True the agent reads/write
        with a perior of DEFAULT_CLOCK
    :ivar ~.pullMode: specifies how the interface should behave if none drives it
        can be (1: pull-up, 0: pull-down, None: disconnected)
    """

    def __init__(self,
                 sim: HdlSimulator,
                 intf: TristateSignal,
                 rst: Tuple["RtlSignal", bool]):
        """
        :param intf: tuple (i signal, o signal, t signal)
            as present in tristate interface
        :note: t signal controls if the output should be connected,
            if 't'=0 the 'o' does not have effect
        """
        super(TristateAgent, self).__init__(sim, intf, rst)
        self.i, self.o, self.t = intf.i, intf.o, intf.t
        self.data = deque()
        # can be (1: pull-up, 0: pull-down, None: disconnected)
        self.pullMode = 1  # type: Union[1, 0, None]
        self.selfSynchronization = True
        self.collectData = True

    def monitor(self):
        """
        The evaluate a tristate 'i' value from 'o' and 't'
        and optionaly store it.
        One step.
        """
        yield WaitCombRead()
        # read in pre-clock-edge
        t = self.t.read()
        o = self.o.read()
        sim = self.sim
        if self.pullMode is not None and sim.now > 0:
            try:
                t = int(t)
            except ValueError:
                raise AssertionError(
                    sim.now, self.t, "Invalid value on tristate interface => ioblock would burn")
            try:
                o = int(o)
            except ValueError:
                raise AssertionError(
                    sim.now, self.o, "Invalid value on tristate interface => ioblock would burn")

            if self.pullMode == o:
                raise AssertionError(
                    sim.now, self.o, "Can not set value to a same as pull up,"
                    " because others may try to set it to oposite => ioblock would burn")

        if t:
            v = o
        else:
            v = self.pullMode

        last = self.i.read()
        try:
            last = int(last)
        except ValueError:
            last = None

        yield WaitWriteOnly()
        self.i.write(v)

        if self.collectData and sim.now > 0:
            yield WaitCombRead()
            if self.notReset():
                self.data.append(v)

    def getMonitors(self):
        return [self.onTWriteCallback(), ]

    def onTWriteCallback(self):
        while True:
            yield Edge(self.t, self.o)
            if self.getEnable():
                # if we are this signal was update by change of some memory we can not write in this
                # time slot and we have to wait for another
                if self.sim._current_time_slot.write_only is DONE:
                    yield Timer(1)
                yield from self.monitor()

    def _write(self, val):
        """
        Update value on interface.

        :type val: Union[int, NOP]
        """
        if val is NOP:
            # control now has slave
            t = 0
            o = self.pullMode
        else:
            # control now has this agent
            t = 1
            o = val

        self.t.write(t)
        self.o.write(o)

    def _read(self):
        """
        :return: actual value on interface
        """
        return self.i.read()

    def driver(self):
        """
        Drive 'o' and 't' from data buffer.
        One step if not selfSynchronization else infinite loop.
        """
        while True:
            yield WaitWriteOnly()
            if self.data:
                o = self.data.popleft()
                if o == NOP:
                    t = 0
                    o = 0
                else:
                    t = 1
                self.o.write(o)
                self.t.write(t)

            if self.selfSynchronization:
                yield Timer(CLK_PERIOD)
            else:
                break

class TristateClkAgent(TristateAgent):
    """
    Agent for tri-state interface which generates clock signal
    and ignores all other components which are trying to drive this clk signal
    """

    def __init__(self, sim: HdlSimulator, intf, rst: Tuple["RtlSignal", bool]):
        super(TristateClkAgent, self).__init__(sim, intf, rst)
        self.period = CLK_PERIOD
        self.collectData = False

    def driver(self):
        o = self.o
        high = self.pullMode
        low = not self.pullMode
        halfPeriod = self.period // 2

        yield WaitWriteOnly()
        o.write(low)
        self.t.write(1)

        while True:
            yield Timer(halfPeriod)
            yield WaitWriteOnly()
            o.write(high)

            yield Timer(halfPeriod)
            yield WaitWriteOnly()
            o.write(low)


"""
# Filling

/usr/bin/spark-submit \
    --master yarn \
    --deploy-mode client \
/mnt/snpfinder-pipeline/Utilities/write_project_parquet.py \
    --job_id 22 \
    --species rice \
    --refasta Japonica.v7.1 \
    --project_path D461 \
    --bam D4610002?D4610001?D4610004?D4610003?D4610005?D4610006?D4610007?D4610008?D4610009?D4610010

"""

import boto3
import json
import math
from optparse import OptionParser
from pyspark import SparkContext, SparkConf
from pyspark.sql import SQLContext
from pyspark.sql.functions import array, coalesce, udf
from pyspark.sql.types import *
from subprocess import check_call
import time
import uuid
import logging

S3_CLIENT = boto3.client(service_name="s3", region_name="us-east-1")

logging.basicConfig(format='%(asctime)s %(levelname)s %(name)s: %(message)s', level=logging.INFO)
logger = logging.getLogger('snpfinder-pipeline:write_project_parquet')

# Get command line arguments
def parse_args():
    usage = "usage: %prog [options]"
    parser = OptionParser(usage=usage)
    parser.add_option("", "--job_id", dest="job_id",
                  help="required EMR job id")
    parser.add_option("", "--species", dest="species",
                  help="Common name of the species associated with the reference assembly")
    parser.add_option("-f","--refasta",dest="refasta",
                  help="The reference fasta filename, needs to have fai index file with it.")
    parser.add_option("", "--job_type", dest="job_type",
                  help="Optional parameter to indicate whether the job type is pipeline or discovery")
    parser.add_option("", "--call_type", dest="call_type",
                  help="Optional parameter to indicate whether the call type is fillin or snarping")
    parser.add_option("", "--project_path", dest="project_path",
                  help="Required path to save the project to")
    parser.add_option("-b","--bam",dest="bam",
                  help="List of sample IDs")

    (options, args) = parser.parse_args()
    if (not options.job_id) \
      or (not options.species) \
      or (not options.refasta) \
      or (not options.project_path) \
      or (not options.bam):
        parser.print_help()
        sys.exit(1)

    return options, args

def get_project_size(bucket, prefix):
    """Returns the total size of all Parquet files stored in the passed prefix/
    directory.

    Args:
        bucket (str): The bucket to search for the prefix.

        prefix (str): The S3 key prefix to search on.

    Returns:
        int. The total size (in bytes) of the Parquet files under the prefix/
        directory.

    """
    paginator = S3_CLIENT.get_paginator("list_objects_v2")
    operation_parameters = {
        "Bucket": bucket,
        "Prefix": "{}/".format(prefix)
    }
    page_iterator = paginator.paginate(**operation_parameters)
    project_size = 0
    for page in page_iterator:
        print("Next page")
        for item in page["Contents"]:
            key = item["Key"]
            ext = key.split(".")[-1]
            size = item["Size"]
            if ext == "parquet":
                project_size += size
    return project_size

# Function to determine whether an INDEL call meets threshold (Filling INDEL calls will not)
def validate_type(data):
    current_call = data[0]
    type_call = data[1]
    coverage = data[2]
    purity = data[3]

    if current_call == 'INDEL':
        new_call =  'INDEL'
    else:
        if type_call == 'SNP':
            new_call =  'SNP'
        else:
            # Empty record
            if type_call is None:
                new_call =  'SNP'
            elif (float(coverage) >= 6.0) and (float(purity) >= 0.98):
                new_call =  'INDEL'
            else:
                new_call =  'SNP'

    return new_call

# Function to select the Locus type from the intermediate project files
def select_intermediate_type(data):
    current_call = data[0]
    type_call = data[1]

    if (data[0] == 'INDEL') or (data[1] == 'INDEL'):
        return 'INDEL'
    else:
        return 'SNP'

# Function to return valid loci
# DataFrame has format ['seq_id', 'position', 'gen_seq_id', 'gen_position', 'type', 'refbase', 'basecall', 'coverage',  'purity',  'alleles']
def get_valid_snps(record):
    # Convert RDD row object to list
    record = [x for x in record]

    # Get columns of interest
    sequence = record[0]
    position = int(record[1])
    reference = record[5]

    # Extract the allele section
    data = record[6:]
    basecalls = data[0::4]

    # Get list of unique allele values, filtering out Nones from outer join
    alleles = [x for x in set(basecalls) if ((x is not None) and (len(x.split('/')) == 1))]

    # If there is only one unique SNP call, see if it matches the reference
    # If it does, it is an invalid artifact of how DELs are called in Samtools
    if (len(alleles) == 1) and (alleles[0] == reference):
        return None
    elif len(alleles) == 0:
        return None
    else:
        return (sequence, position)

def main():
    # Get command line options
    options, args = parse_args()

    # Assign command line parameters to variables
    options, args = parse_args()
    species = options.species
    refasta = options.refasta
    job_id = options.job_id
    project_path = options.project_path
    sample_list = (options.bam).split('?')

    # Set configuration for the cluster
    conf = SparkConf().setAppName("Write Project Parquet")
    sc = SparkContext(conf = conf)
    sqlContext = SQLContext(sc)

    # Load S3 intermediate and output file paths
    with open('/mnt/paths.json', 'r') as f:
        s3_paths = json.load(f)
        bucket = s3_paths["bucket"]
        s3_intermediate_path = '{}/{}'.format(s3_paths["bucket"], s3_paths['intermediate_directory'])
        s3_output_path = '{}/{}'.format(s3_paths["bucket"], s3_paths['output_directory'])

    # UDF for setting type to INDEL if any sample has the type INS or DEL
    choose_type = udf(lambda x: validate_type(x), returnType=StringType())

    # UDF for setting type to INDEL if any intermediate project file has an INDEL at a locus
    choose_intermediate_type = udf(lambda x: select_intermediate_type(x), returnType=StringType())

    # DataFrame has format ['seq_id', 'position', 'gen_seq_id', 'gen_position', 'type', 'refbase', 'basecall', 'coverage',  'purity',  'alleles']
    dup_count = 0

    # Get the cumulative size of the Project's Samples
    # (used later to calculat the number of partitions for the final file)
    project_size = 0
    bucket = s3_paths["bucket"].replace('s3://', '')
    for sample in sample_list:
        prefix = '{}/{}/{}/parquet'.format(s3_paths['intermediate_directory'], job_id, sample)
        project_size += get_project_size(bucket, prefix)

    num_partitions = max(1, int(math.floor(float(project_size) / (128 * 1024 ** 2))))
    size_gb = round(float(project_size) / (1024 ** 3), 2)
    logger.info('Sample Parquet files total {} GB. Project Parquet file will be broken into {:,} partitions.'.format(size_gb, num_partitions))

    # If we try to join too many samples at once, we will get a java.lang.StackOverflowError
    # For every 50 samples, write an intermediate Parquet file
    sample_bucket = {}
    bucket_index = 0
    for index, sample in enumerate(sample_list):
        # Assign the sample to an intermediate file bin
        if index % 50 == 0:
            bucket_index += 1

        if bucket_index in sample_bucket:
            sample_bucket[bucket_index].append(sample)
        else:
            sample_bucket[bucket_index] = [sample]

    # Write intermediate Parquet files_to_send
    for key in sample_bucket:
        for index, sample in enumerate(sample_bucket[key]):
            logger.info('Joining {}'.format(sample))
            input_path = '{}/{}/{}/parquet'.format(s3_intermediate_path, job_id, sample)

            # Load each Parquet files into DataFrame and join to other files in the project
            if index == 0:
                df = sqlContext.read.parquet(input_path)
                column_names = df.columns
                cov_col = column_names[7]
                temp_cov_col = '{}_'.format(cov_col)
                df = (df.withColumn('position_', df['position'].cast(IntegerType()))
                      .withColumn(temp_cov_col, df[cov_col].cast(IntegerType()))
                      .drop('position', cov_col)
                      .withColumnRenamed('position_', 'position')
                      .withColumnRenamed(temp_cov_col, cov_col)
                ).select(column_names)
            else:
                df2 = sqlContext.read.parquet(input_path)
                df2_column_names = df2.columns
                cov_col = df2_column_names[7]
                temp_cov_col = '{}_'.format(cov_col)
                df2 = (df2.withColumn('position_', df2['position'].cast(IntegerType()))
                        .withColumn(temp_cov_col, df2[cov_col].cast(IntegerType()))
                        .drop('position', cov_col)
                        .withColumnRenamed('position_', 'position')
                        .withColumnRenamed(temp_cov_col, cov_col)
                    ).select(df2_column_names)

                # Join DataFrames, coalesce columns, and drop duplicates
                df = (df.join(df2, (df.seq_id==df2.seq_id) & (df.position==df2.position), 'outer')
                        .withColumn('seq_id_', coalesce(df.seq_id, df2.seq_id))
                        .withColumn('position_', coalesce(df.position, df2.position))
                        .withColumn('gen_seq_id_', coalesce(df.gen_seq_id, df2.gen_seq_id))
                        .withColumn('gen_position_', coalesce(df.gen_position, df2.gen_position))
                        .withColumn('type_', choose_type(array(df.type, df2.type, df2[df2_column_names[7]], df2[df2_column_names[8]])))
                        .withColumn('refbase_', coalesce(df.refbase, df2.refbase))
                        .drop('seq_id', 'position', 'gen_seq_id', 'gen_position', 'type', 'refbase')
                        .withColumnRenamed('seq_id_', 'seq_id')
                        .withColumnRenamed('position_', 'position')
                        .withColumnRenamed('gen_seq_id_', 'gen_seq_id')
                        .withColumnRenamed('gen_position_', 'gen_position')
                        .withColumnRenamed('type_', 'type')
                        .withColumnRenamed('refbase_', 'refbase')
                    )

                # Check for duplicate column names
                # (Using the sample name should reduce the frequency of this occuring, but it probably will)
                if df2_column_names[6] in column_names:
                    dup_count += 1
                    column_names += [
                        df2_column_names[6].replace('basecall', '{}_basecall'.format(dup_count)),
                        df2_column_names[7].replace('coverage', '{}_coverage'.format(dup_count)),
                        df2_column_names[8].replace('purity', '{}_purity'.format(dup_count)),
                        df2_column_names[9].replace('alleles', '{}_alleles'.format(dup_count))
                    ]
                else:
                    # Append columns to column name list
                    column_names += df2_column_names[6:10]

            # Reorder DataFrame columns
            df = df.select(column_names)

        # Save intermediate DataFrame to file
        logger.info('Writing intermediate project file {}...'.format(key))
        hdfs_path = '/user/warehouse/{}'.format(str(uuid.uuid1()))
        temp_path = '{}/{}/t{}'.format(s3_intermediate_path, job_id, key)
        """
        df.write.parquet(hdfs_path, mode='overwrite')

        # Delete any existing project from S3, otherwise you'll end up with data
        # from multiple parquet files mixed together
        cmd = "aws s3 rm {} --recursive --only-show-errors".format(temp_path)
        check_call(cmd, shell=True)

        # Copy from HDFS to S3
        cmd = 's3-dist-cp --src {} --dest {}'.format(hdfs_path, temp_path)
        check_call(cmd, shell=True)

        cmd = 'hdfs dfs -rm -r {}'.format(hdfs_path)
        check_call(cmd, shell=True)
        """
        df.write.parquet(temp_path, mode='overwrite')

    # Get the list of sequence ID's for the assembly
    with open('/mnt/sequences.json', 'r') as f:
        seqs = list(json.load(f))
        num_seq = len(seqs)

    # Join the intermediate DataFrames into a single Project-level DataFrame
    for key in sample_bucket:
        temp_path = '{}/{}/t{}'.format(s3_intermediate_path, job_id, key)
        logger.info('Merging intermediate project file {}...'.format(key))
        if key == 1:
            df = sqlContext.read.parquet(temp_path)
            column_names = df.columns
        else:
            df2 = sqlContext.read.parquet(temp_path)
            column_names += df2.columns[6:]

            df = (df.join(df2, (df.seq_id==df2.seq_id) & (df.position==df2.position), 'outer')
                    .withColumn('seq_id_', coalesce(df.seq_id, df2.seq_id))
                    .withColumn('position_', coalesce(df.position, df2.position))
                    .withColumn('gen_seq_id_', coalesce(df.gen_seq_id, df2.gen_seq_id))
                    .withColumn('gen_position_', coalesce(df.gen_position, df2.gen_position))
                    .withColumn('type_', choose_intermediate_type(array(df.type, df2.type)))
                    .withColumn('refbase_', coalesce(df.refbase, df2.refbase))
                    .drop('seq_id', 'position', 'gen_seq_id', 'gen_position', 'type', 'refbase')
                    .withColumnRenamed('seq_id_', 'seq_id')
                    .withColumnRenamed('position_', 'position')
                    .withColumnRenamed('gen_seq_id_', 'gen_seq_id')
                    .withColumnRenamed('gen_position_', 'gen_position')
                    .withColumnRenamed('type_', 'type')
                    .withColumnRenamed('refbase_', 'refbase')
                )

    # Filter out extraneous sequences
    df = df[df.seq_id.isin(seqs)]
    df = df.select(column_names)
    """
    if options.call_type != 'snarping':
        # Filter out positions where all basecalls match the reference assembly
        logger.info('Filtering out invalid SNPs')

        # Convert the DataFrame to RDD
        rdd = df.rdd

        # Look for positions where there is only one unique allele call and it matches the reference assembly
        # (The extra alleles are an artifact of how Samtools calls DELETEs, which is one base pair prior to the deleted base)
        valid_SNPs = rdd.map(get_valid_snps).collect()
        valid_SNPs = [x for x in valid_SNPs if x is not None]

        # Create a DataFrame from the lsit of valid SNP positions
        schema = StructType([
            StructField("seq_id", StringType(), True),
            StructField("position", IntegerType(), True)
        ])
        valid_df = sqlContext.createDataFrame(valid_SNPs, schema)

        # Filter out invalid positions (ones with no actual SNPs) by inner joining the project-level DataFrame with the list of valid positions
        df = (df.join(valid_df, (df.seq_id==valid_df.seq_id) & (df.position==valid_df.position), 'inner')
                .withColumn('seq_id_', coalesce(df.seq_id, valid_df.seq_id))
                .withColumn('position_', coalesce(df.position, valid_df.position))
                .drop('seq_id', 'position')
                .withColumnRenamed('seq_id_', 'seq_id')
                .withColumnRenamed('position_', 'position')
            )

        # Reorder DataFrame columns
        df = df.select(column_names)
    """
    # Write DataFrame to Parquet, sorting on 'position' and partitioning on 'seq_id'
    logger.info('Writing project to Parquet file')

    if options.job_type and (options.job_type.lower() == 'discovery'):
        # Save parquet file to the intermediate_files folder
        output_path = '{}/{}/{}/{}/{}'.format(s3_intermediate_path, options.job_id, species, refasta, project_path)
    else:
        # Save parquet file to the long_term_storage folder
        output_path = '{}/{}/{}/{}'.format(s3_output_path, species, refasta, project_path)

    df = df.sort('seq_id', 'position')

    # Check to see whether we should partition based on Sequence ID
    # (The Parquet writer will fail if there are too many Sequence IDs to partion by)
    if num_seq > 500:
        num_partitions = int(math.floor(float(num_partitions) / 20))
    else:
        num_partitions = int(math.floor(float(num_partitions) / num_seq))

    if num_partitions < 1:
        num_partitions = 1

    # Write to HDFS (saves a lot of time for references with many contigs/scaffolds)
    logger.info('Partitioning Parquet file by Sequence ID ({:,} partitions per sequence)'.format(num_partitions))
    """
    # NOTE: s3distcp is only installed on the master, not the workers, so this won't work
    # running in cluster mode without installing s3distcp on the workers
    # TODO: Add step in this program to download s3distcp from S3

    hdfs_path = '/user/warehouse/{}'.format(str(uuid.uuid1()))
    df.repartition(num_partitions).write.parquet(hdfs_path, mode='overwrite', partitionBy='seq_id')

    # Delete any existing project from S3, otherwise you'll end up with data
    # from multiple parquet files mixed together
    cmd = "aws s3 rm {} --recursive --only-show-errors".format(output_path)
    check_call(cmd, shell=True)

    # Copy from HDFS to S3
    cmd = 's3-dist-cp --src {} --dest {}'.format(hdfs_path, output_path)
    check_call(cmd, shell=True)

    cmd = 'hdfs dfs -rm -r {}'.format(hdfs_path)
    check_call(cmd, shell=True)
    """
    df.repartition(num_partitions).write.parquet(output_path, mode='overwrite', partitionBy='seq_id')

# Execute main function
if __name__ == "__main__":
    start_time = time.time()
    main()
    logger.info("Total time {0:.4f} seconds".format(time.time() - start_time))


from numpy import *
from matplotlib.pyplot import *
from numpy.polynomial import Polynomial
from scipy import optimize

class RemezApprox:
    def __init__(self,f):
        if not inspect.isfunction(f):
            raise TypeError(f'Expected argument should be of type function, not {type(f)}')
        self.func=f

    def __createPolynom(self,x):
            n=self.n
            M=zeros((n+2,n+2)) #Skapa en n+2 x n+2 matris fylld med nollor
            for i in range(n+2): #Första kolumnen bara ettor? Fyll matrisen rad för rad, förutom sista kolumnen.
                    for j in range(n+1):
                        M[i][j]=x[i]**j

            for i in range(n+2): #Sista kolumnen, fylld med 1 och -1.
                M[i][-1]=(-1)**i
            F=self.func(x) #Kända funktionsvärden för testfunktionen i värdena x0,x1,x2 ... i vårt intervall.
            self.Pcoeff= linalg.solve(M,F) #Löser linjära ekvationsystemet. Får ut koefficienterna för polynomet, där sista elementet är h.
            p=Polynomial(self.Pcoeff[:-1]) #Polynomial skapar ett polynom objekt med värdena i Pcoeff förutom sista värdet 'h'.
            return p

    def remez(self,n,a,b, maxIteration = 50, tolerance = 1e-6):
        if not isinstance(n, int):
            raise TypeError(f'The polynomial degree needs to be of type Int, not {type(n)}')
        if not isinstance(a, (int, float)):
            raise TypeError(f'Reference value needs to be of type Int/Float, not {type(a)}')
        if not isinstance(b, (int, float)):
            raise TypeError(f'Reference value needs to be of type Int/Float, not {type(b)}')
        if not isinstance(maxIteration, int):
            raise TypeError(f'Number of max iterations needs to be of type Int, not {type(maxIteration)}')
        if not isinstance(tolerance, (int, float)):
            raise TypeError(f'The tolerance needs to be of type Int/Float, not {type(tolerance)}')

        self.tol = tolerance
        self.maxiter = maxIteration
        self.n=n

        x=array(linspace(a,b,n+2))
        self.x=x

        t=linspace(a,b,int(1e4))
        self.t=t

        f = self.func
        p = self.__createPolynom(x)

        counter=0
        Pcoeff=self.Pcoeff

        while abs(Pcoeff[-1]-max(abs(p(t)-f(t))))>self.tol and counter<self.maxiter:
            counter+=1
            self.p = self.__createPolynom(x) #Skapar nytt polynom med efter bytt ut en punkt i referensen.
            L = [(abs(f(i)-p(i))) for i in t] #Används för att hitta ett max i ett tätare intervall istället för derivera.
            max_value = max(L)
            my = t[L.index(max_value)]

        #Steg 4.
            if my<x[0]:
                if sign(f(my)-p(my)) == sign(f(x[0])-p(x[0])):
                    x[0]=my
                else:
                    x[-1]=my

            if my>x[-1]:
                if sign(f(my)-p(my)) == sign(f(x[-1])-p(x[-1])):
                    x[-1]=my
                else:
                    x[0]=my

            else:
                for i in range(len(x)):
                    if (my<=x[i]) :
                        x_i=i-1
                        break
                if sign(f(my)-p(my)) == sign(f(x[x_i])-p(x[x_i])):
                    x[x_i] = my
                else:
                    x[x_i+1] = my

        # self.polynomial = p(x) # Kommer inte till använding i koden?
        # self.counter=counter # Kommer inte till använding i koden?

    def plotRemez(self):
        t=self.t
        figure(0)
        plot(t,self.p(t))
        plot(t,self.func(t))

    def plotError(self):
        x=self.x
        t=self.t
        figure(1)
        plot(t,(abs(self.p(t)-self.func(t)))) #Plottar error funktionen.
        yscale('log')
        plot(x,1+0*x,'*')

def f(x): #definera en testfunktion
    return sin(x)
G=RemezApprox(f)
G.remez(10,0,10) #degree,a,b
G.plotRemez()
G.plotError()

Uppdaterad remez klass med felhantering.py


import socket
import time

class TcpClient(object):

    def client_func(self):
        self.tcp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        server_address = ('192.168.1.187', 7788)
        self.tcp_socket.connect(server_address)
        while True:
            nowtime = int(time.time())
            time_local = time.localtime(nowtime)
            now_time = time.strftime('%Y%m%d%H%M%S', time_local)
            result = str(now_time)
            self.tcp_socket.sendall(result.encode('utf-8'))
            receive_data = self.tcp_socket.recv(1024)
            print('当前连接时间及次数:', receive_data.decode(encoding='utf-8'))
            yield

    # def standy_client(self):
    #     tcp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    #     server_address = ('192.168.1.187', 7788)
    #     tcp_socket.connect(server_address)
    #     while True:
    #         nowtime = int(time.time())
    #         time_local = time.localtime(nowtime)
    #         now_time = time.strftime('%Y%m%d%H%M%S', time_local)
    #         result = str(now_time)
    #         tcp_socket.sendall(result.encode('utf-8'))
    #         receive_data = tcp_socket.recv(1024)
    #         print('当前连接时间及次数:', receive_data.decode(encoding='utf-8'))
    #         yield

def start_client(mainfunc,timesleep,times,moinfunc,confunc):
    count = 0

    while True:
        try:
            next(mainfunc)
        except Exception:
            count += 1
            print(count)
        time.sleep(timesleep)
        if count > times:
            msg = '与服务器断开连接，请重启服务器与客户端'
            print(msg)
            break

            # start_client(TcpClient().client_func(),standyfunc,timesleep,times,moinfunc,confunc)

                # moinfunc()
    # while True:
    #     try:
    #         print('1111')
    #         # confunc()
    #         next(standyfunc)
    #         print('3333')
    #     except Exception:
    #         print('2222')

        time.sleep(timesleep)


import os
import requests
from flask import jsonify, request
from marshmallow import ValidationError
from openpatch_core.database import db
from openpatch_core.rabbitmq import rabbit
from openpatch_core.jwt import get_jwt_claims, jwt_required
from openpatch_itembank.api.v1 import api, errors
from openpatch_itembank.api.v1.schemas.item import ITEM_SCHEMA, ITEMS_SCHEMA
from openpatch_itembank.api.v1.schemas.item_version import (
    ITEM_VERSION_SCHEMA,
    ITEM_VERSIONS_SCHEMA,
)
from openpatch_itembank.api.v1.schemas.member import MEMBER_SCHEMA
from openpatch_itembank.models.item import Item
from openpatch_itembank.models.item_version import ItemVersion
from openpatch_itembank.models.test_version import TestItem
from openpatch_itembank.models.member import Member
from openpatch_itembank.models.privacy import Privacy
from sqlalchemy.exc import DataError, StatementError

BASE_URL = "/items"

@api.route(BASE_URL, methods=["POST", "GET"])
def items():
    if request.method == "POST":
        return post_items()
    return get_items()

@jwt_required
def post_items():
    """
    @api {post} /v1/items Post items
    @apiVersion 1.0.0
    @apiName PostItems
    @apiGroup Items

    @apiUse jwt

    @apiParam {String{1..255}} name Name
    @apiParam {String} private_description Private hidden description
    @apiParam {String} public_description Public description
    @apiParam {String} source_reference Reference to the original source
    @apiParam {Boolean} public Is the item visible to all user
    @apiParam {String{5}} language Country Code

    @apiSuccess {Number} item_id The item id.

    @apiUse errors_no_json
    @apiUse errors_invalid_json

    @apiPermission admin
    @apiPermission author
    """
    jwt_claims = get_jwt_claims()

    item_json = request.get_json()
    if not item_json:
        return errors.no_json()

    # item is created by current member
    member = Member.get_or_create(jwt_claims)

    try:
        item = ITEM_SCHEMA.load(item_json, session=db.session)
        item.member = member
    except ValidationError as e:
        return errors.invalid_json(e.messages)

    db.session.add(item)
    db.session.commit()

    item_version = ItemVersion(
        latest=True,
        member=member,
        status="draft",
        item=item,
        version=1,
        tasks=[],
        version_message="Initial",
    )
    db.session.add(item_version)
    db.session.commit()

    if item.privacy == Privacy.public:
        rabbit.publish_activity(
            "create",
            {"id": item.id, "type": "item", "data": ITEM_SCHEMA.dump(item)},
            MEMBER_SCHEMA.dump(member),
        )

    return jsonify({"item_id": item.id})

def get_items():
    """
    @api {get} /v1/items Get items
    @apiVersion 1.0.0
    @apiName GetItems
    @apiGroup Items

    @apiUse jwt
    @apiUse elastic_query

    @apiSuccess {Object[]} items
    @apiSuccess {Number} items.id The item id.
    @apiSuccess {String{1..255}} items.name Name
    @apiSuccess {String} items.private_description Private hidden description
    @apiSuccess {String} items.public_description Public description
    @apiSuccess {String} items.source_reference Reference to the original source
    @apiSuccess {Boolean} items.public Is the item visible to all user
    @apiSuccess {String{5}} items.language Country Code
    """
    jwt_claims = get_jwt_claims(optional=True)

    item_query, count, page = Item.elastic_query(request.args.get("query", "{}"))

    if not jwt_claims:
        item_query = item_query.filter(Item.privacy == Privacy.public)
    elif jwt_claims.get("role") != "admin":
        # select only items created by the member and permitted items
        item_query = item_query.filter(
            db.or_(
                Item.privacy == Privacy.public, Item.member_id == jwt_claims.get("id")
            )
        )

    count = item_query.count()
    item_query = page(query=item_query)
    items = item_query.all()

    return jsonify({"items": ITEMS_SCHEMA.dump(items), "items_count": count}), 200

@api.route(BASE_URL + "/<item_id>", methods=["PUT", "GET", "DELETE"])
def item(item_id):
    if request.method == "PUT":
        return put_item(item_id)
    elif request.method == "DELETE":
        return delete_item(item_id)
    return get_item(item_id)

@jwt_required
def put_item(item_id):
    """
    @api {put} /v1/items/:id Put item
    @apiVersion 1.0.0
    @apiName PutItem
    @apiGroup Items

    @apiUse jwt

    @apiParam {Number} id Item id
    @apiParam {String{1..255}} name Name
    @apiParam {String} private_description Private hidden description
    @apiParam {String} public_description Public description
    @apiParam {String} source_reference Reference to the original source
    @apiParam {Boolean} public Is the item visible to all user
    @apiParam {String{5}} language Country Code

    @apiSuccess {Number} item_id Item id

    @apiUse errors_no_json
    @apiUse errors_invalid_json
    @apiUse errors_resource_not_found
    @apiUse errors_access_not_allowed

    @apiPermission admin
    @apiPermission author
    """
    jwt_claims = get_jwt_claims()
    member = Member.get_or_create(jwt_claims)

    item_json = request.get_json()
    if not item_json:
        return errors.no_json()

    try:
        item = Item.query.get(item_id)
    except StatementError:
        return errors.resource_not_found()

    if not item:
        return errors.resource_not_found()

    if not item.permitted_write(jwt_claims):
        return errors.access_not_allowed()

    if "versions" in item_json:
        del item_json["versions"]

    if "member" in item_json:
        del item_json["member"]

    if (
        item.used_in_tests > 0
        and item.privacy != Privacy.private
        and item_json["privacy"] == "private"
    ):
        del item_json["privacy"]

    try:
        result = ITEM_SCHEMA.load(
            item_json, session=db.session, instance=item, partial=True
        )
        result.member = member
    except ValidationError as e:
        return errors.invalid_json(e.messages)

    if item.privacy == Privacy.public:
        rabbit.publish_activity(
            "update",
            {"id": item.id, "type": "item", "data": ITEM_SCHEMA.dump(item)},
            MEMBER_SCHEMA.dump(member),
        )

    db.session.commit()

    return jsonify({"item_id": item.id}), 200

@jwt_required
def delete_item(item_id):
    """
    @api {delete} /v1/items/:id Delete item
    @apiVersion 1.0.0
    @apiName DeleteItem
    @apiGroup Items

    @apiUse jwt

    @apiParam {Number} id Item id

    @apiUse errors_resource_not_found
    @apiUse errors_access_not_allowed
    @apiUse errors_item_is_in_use

    @apiPermission admin
    @apiPermission author
    """
    jwt_claims = get_jwt_claims()
    member = Member.get_or_create(jwt_claims)

    try:
        item = Item.query.get(item_id)
    except StatementError:
        return errors.resource_not_found()

    if not item:
        return errors.resource_not_found()

    if not item.permitted_write(jwt_claims):
        return errors.access_not_allowed()

    # check if item is used in a test
    test_item = TestItem.query.filter_by(item=item).count()
    if test_item > 0:
        return errors.item_is_in_use()

    for version in item.versions:
        db.session.delete(version)

    db.session.delete(item)

    if item.privacy == Privacy.public:
        rabbit.publish_activity(
            "delete",
            {"id": item.id, "type": "item", "data": ITEM_SCHEMA.dump(item)},
            MEMBER_SCHEMA.dump(member),
        )

    db.session.commit()

    return jsonify({}), 200

def get_item(item_id):
    """
    @api {get} /v1/items/:id Get item
    @apiVersion 1.0.0
    @apiName GetItem
    @apiGroup Items

    @apiUse jwt

    @apiParam {Number} id Item id

    @apiSuccess {Object} item
    @apiSuccess {Number} item.id The item id.
    @apiSuccess {String{1..255}} item.name Name
    @apiSuccess {String} item.private_description Private hidden description
    @apiSuccess {String} item.public_description Public description
    @apiSuccess {String} item.source_reference Reference to the original source
    @apiSuccess {Boolean} item.public Is the item visible to all user
    @apiSuccess {String{5}} item.language Country Code

    @apiUse errors_resource_not_found
    @apiUse errors_access_not_allowed
    """
    jwt_claims = get_jwt_claims(optional=True)

    try:
        item = Item.query.get(item_id)
    except StatementError:
        return errors.resource_not_found()

    if not item:
        return errors.resource_not_found()

    if not item.permitted_read(jwt_claims):
        return errors.access_not_allowed()

    return jsonify({"item": ITEM_SCHEMA.dump(item)})

@api.route(BASE_URL + "/<item_id>/remix", methods=["POST"])
@jwt_required
def remix_item(item_id):
    jwt_claims = get_jwt_claims()
    member = Member.get_or_create(jwt_claims)

    try:
        item = Item.query.get(item_id)
    except StatementError:
        return errors.resource_not_found()

    if not item:
        return errors.resource_not_found()

    if not item.permitted_read(jwt_claims):
        return errors.access_not_allowed()

    new_item = Item(
        name=item.name + " (Remix)",
        public_description=item.public_description,
        public_description_text=item.public_description_text,
        source_reference=item.source_reference,
        privacy=item.privacy,
        language=item.language,
        member=member,
    )

    db.session.add(new_item)

    latest_item_version = ItemVersion.get_latest(item_id)
    if not latest_item_version:
        latest_item_version = ItemVersion.get_draft(item_id)
    if not latest_item_version:
        return errors.resource_not_found()
    new_item_version = latest_item_version.copy()
    new_item_version.member = member
    new_item_version.version = 1
    new_item_version.version_message = "remix"
    new_item_version.status = "draft"
    new_item_version.latest = True
    new_item_version.item = new_item

    item.remixes.append(new_item)

    db.session.add(new_item_version)
    db.session.commit()

    if new_item.privacy == Privacy.public:
        rabbit.publish_activity(
            "create:remix",
            {"id": item.id, "type": "item", "data": ITEM_SCHEMA.dump(item)},
            MEMBER_SCHEMA.dump(member),
        )

    return jsonify({"item_id": new_item.id})

@api.route(BASE_URL + "/versions/evaluate", methods=["POST"])
@jwt_required
def evaluate_item_version():
    """
    @api {get} /v1/items/versions/evaluate Evaluate Item Version
    @apiVersion 1.0.0
    @apiName PostVersionsEvaluate
    @apiGroup Items

    @apiUse jwt

    @apiParam {Object[]} tasks Array of tasks
    @apiParam {String} tasks.format_type Format Type
    @apiParam {Object} solutions Object of solutions ids need to match task array index

    @apiSuccess {Object} result.

    @apiUse errors_format_service_down
    @apiUse errors_invalid_json
    """
    evaluate_json = request.get_json()

    format_url = os.getenv("OPENPATCH_FORMAT_SERVICE")
    if not format_url:
        return errors.format_service_down()

    try:
        r = requests.post("%s/v1/evaluate-batch" % format_url, json=evaluate_json)
        if r.status_code >= 500:
            return errors.format_service_down()
        elif r.status_code >= 400:
            return errors.invalid_json(r.json().get("details"))

        return jsonify(r.json()), r.status_code, r.headers.items()

    except requests.ConnectionError:
        return errors.format_service_down()

@api.route(BASE_URL + "/<item_id>/versions/<version>", methods=["GET"])
def get_item_version(item_id, version):
    jwt_claims = get_jwt_claims(optional=True)

    item_version_query = ItemVersion.query.filter_by(item_id=item_id)
    if version == "latest":
        item_version_query = item_version_query.filter_by(latest=True)
    elif version == "draft":
        item_version_query = item_version_query.filter_by(status="draft")
    else:
        item_version_query = item_version_query.filter_by(version=version)

    item_version = item_version_query.first()

    if not item_version:
        return errors.resource_not_found()

    if not item_version.permitted_read(jwt_claims):
        return errors.access_not_allowed()

    return jsonify({"item_version": ITEM_VERSION_SCHEMA.dump(item_version)})

@api.route(BASE_URL + "/<item_id>/versions", methods=["GET", "POST", "PUT"])
def item_versions(item_id):
    if request.method == "POST":
        return post_item_versions(item_id)
    elif request.method == "PUT":
        return put_item_versions(item_id)
    return get_item_versions(item_id)

def get_item_versions(item_id):
    """
    @api {get} /v1/items/:id/versions Get item versions
    @apiVersion 1.0.0
    @apiName GetItemVersions
    @apiGroup Items

    @apiUse jwt
    @apiUse elastic_query
    """
    jwt_claims = get_jwt_claims()

    try:
        item = Item.query.get(item_id)
    except StatementError:
        return errors.resource_not_found()

    if not item:
        return errors.resource_not_found()

    if not item.permitted_read(jwt_claims):
        return errors.access_not_allowed()

    item_versions_query, count, page = ItemVersion.elastic_query(
        request.args.get("query", "{}")
    )
    item_versions_query = item_versions_query.filter_by(item=item)
    count = item_versions_query.count()

    if jwt_claims.get("role") != "admin":
        # select only items created by the member and permitted items
        item_versions_query = item_versions_query.filter(
            db.or_(
                Item.privacy != Privacy.private, Item.member_id == jwt_claims.get("id")
            )
        )

    item_versions_query = page(query=item_versions_query)
    item_versions = item_versions_query.all()

    return jsonify(
        {
            "item_versions": ITEM_VERSIONS_SCHEMA.dump(item_versions),
            "item_versions_count": count,
        }
    )

@jwt_required
def put_item_versions(item_id):
    jwt_claims = get_jwt_claims()
    member = Member.get_or_create(jwt_claims)

    item_version_json = request.get_json()

    if not item_version_json:
        return errors.no_json()

    item_version_json["status"] = "draft"
    if "id" in item_version_json:
        del item_version_json["id"]
    if "version" in item_version_json:
        del item_version_json["version"]
    if "version_message" in item_version_json:
        del item_version_json["version_message"]
    if "latest" in item_version_json:
        del item_version_json["latest"]
    if "member" in item_version_json:
        del item_version_json["member"]

    item_version = ItemVersion.get_draft(item_id)
    if not item_version:
        return errors.resource_not_found()

    try:
        item = Item.query.get(item_id)
    except StatementError:
        return errors.resource_not_found()

    if not item.permitted_write(jwt_claims):
        return errors.access_not_allowed()

    for task in item_version_json.get("tasks", []):
        task["format_version"] = 1
        if "item_version" in task:
            del task["item_version"]

    try:
        with db.session.no_autoflush:
            result = ITEM_VERSION_SCHEMA.load(
                item_version_json,
                session=db.session,
                instance=item_version,
                partial=True,
            )
            result.member = member
    except ValidationError as e:
        db.session.rollback()
        return errors.invalid_json(e.messages)

    db.session.commit()

    if item.privacy == Privacy.public:
        rabbit.publish_activity(
            "update:version",
            {"id": item.id, "type": "item", "data": ITEM_SCHEMA.dump(item)},
            MEMBER_SCHEMA.dump(member),
        )

    return jsonify({}), 200

@jwt_required
def post_item_versions(item_id):
    # aka new draft
    jwt_claims = get_jwt_claims()

    item_version_json = request.get_json()
    if not item_version_json:
        return errors.no_json()

    # item version is created by current member
    member = Member.get_or_create(jwt_claims)

    draft_item_version = ItemVersion.get_draft(item_id)
    latest_item_version = ItemVersion.get_latest(item_id)
    if not draft_item_version:
        return errors.resource_not_found()

    new_draft_item_version = draft_item_version.copy()
    draft_item_version.status = "pilot"
    draft_item_version.latest = True
    draft_item_version.version_message = item_version_json["version_message"]
    new_draft_item_version.version_message = "draft"
    new_draft_item_version.version = draft_item_version.version + 1
    new_draft_item_version.member = member

    if latest_item_version:
        latest_item_version.latest = False

    db.session.add(new_draft_item_version)
    db.session.commit()

    item = new_draft_item_version.item
    if item.privacy == Privacy.public:
        rabbit.publish_activity(
            "create:version",
            {"id": item.id, "type": "item", "data": ITEM_SCHEMA.dump(item)},
            MEMBER_SCHEMA.dump(member),
        )

    return jsonify(
        {
            "version": new_draft_item_version.version,
            "item_id": new_draft_item_version.item_id,
        }
    )

@api.route(BASE_URL + "/<item_id>/versions/<version>/status", methods=["PUT"])
@jwt_required
def put_item_version_status(item_id, version):
    json = request.get_json()
    if not json:
        return errors.no_json()

    status = json.get("status")

    if not status or status not in ["pilot", "ready", "faulty"]:
        return errors.invalid_json("Wrong status")

    jwt_claims = get_jwt_claims()

    item = ItemVersion.query.filter_by(item_id=item_id, version=version).first()

    if not item:
        return errors.resource_not_found()

    if not item.permitted_write(jwt_claims) or item.status == "draft":
        return errors.access_not_allowed()

    try:
        item.status = status
        db.session.commit()
    except DataError as e:
        db.session.rollback()
        return errors.invalid_json(e.messages)

    return jsonify({}), 200


import math
import math

def calc_fitness(gene,pr_type):
    summation = 0
    product = 1
    sum_result = 55
    product_result = 3628800
    overall_error = 0
    for i in range(len(gene)):
        if gene[i] == 0:
            summation = summation + (i+1)
        else:
            product = product * (i+1)
    #===========first_fitness_method=============#
    # min_available = 0
    # max_available = 13168184400
    # result = math.sqrt((36-summation)**2 + (360-product)**2)
    # overall_error = (result) / max_available
    #===========third_fitness_method=============#
    # if summation == 0:
    #     summation = 0.001
    # if product == 0:
    #     product = 0.001
    # sum_fitness = 36/summation
    # production_fitness = 360/product
    # overall_error = sum_fitness + production_fitness
    if pr_type == 'max':
        sum_error = abs((sum_result - summation) / sum_result)
        product_error = abs((product_result - product) / product_result)
        overall_error = sum_error + product_error +0.1
    if pr_type == 'min':
        sum_error = abs((sum_result - summation) / sum_result)
        product_error = abs((product_result - product) / product_result)
        overall_error = sum_error + product_error +0.1
        overall_error = 1/overall_error

    return overall_error, summation, product

def fitness_func(population,pr_type):

    fitness_array = []
    cumsum_fitness = 0

    result = []

    for i in range(len(population)):
        fitness, summation, production = calc_fitness(population[i],pr_type)
        control_array = []
        fitness_array.append(fitness)
        control_array.append(fitness)
        control_array.append(summation)
        control_array.append(production)
        result.append(control_array)
        cumsum_fitness = cumsum_fitness + fitness

    best_solution = 0
    best_solution_array = fitness_array[0]
    for i in range(len(fitness_array)):
        if pr_type == 'max':
            if fitness_array[i] > best_solution:
                best_solution = fitness_array[i]
                best_solution_array =population[i]
        if pr_type == 'min':
            if fitness_array[i] < best_solution:
                best_solution = fitness_array[i]
                best_solution_array = population[i]

    average_fitness = cumsum_fitness / len(fitness_array)

    #==============normalizing_fitness_array================#
    # min_fitness = min(fitness_array)
    # max_fitness = max(fitness_array)
    # fitness_array_normalized = []
    # for fitn in fitness_array:
    #     fitness_array_normalized.append((fitn - min_fitness)/(max_fitness - min_fitness))
    # for nor in fitness_array_normalized:
    #     print("normalized_fitness: ",nor)

    selecton_probability = []
    for fit in fitness_array:
        selecton_probability.append(fit/cumsum_fitness)
    for i in range(len(result)):
        print("fitness : ", result[i][0], "summation : ", result[i][1], "prodcut : ",
              result[i][2], "selection_probability : ", selecton_probability[i],"Population : ",population[i])

    # selecton_probability_max = max(selecton_probability)
    # selecton_probability_min = min(selecton_probability)
    # selecton_probability_normalized = []
    # for probability in selecton_probability:
    #     selecton_probability_normalized.append((probability - selecton_probability_min)/(selecton_probability_max - selecton_probability_min))

    return fitness_array, average_fitness, best_solution ,best_solution_array, selecton_probability


import os
import pygame as pg
from .scene import Scene
from ..object.button import Button
from .. import setup
from .. import constants as c
from .. import tools
from..object.ai import airandom
from..object.ai import aiplayer
from..object.ai import aicircle
from..object.ai import aichose

name_width = 130
name_height = 30

class Select(Scene):
    def __init__(self):
        super().__init__()
        self.btn_group.add(Button(self.back, (323, 500), setup.IMG['return0'], setup.IMG['return1']))

        self.ai_names = []
        self.ai_paths = []
        path = os.path.join('game', 'object', 'ai')
        for pic in os.listdir(path):
            name, ext = os.path.splitext(pic)
            if ext.lower() == '.py':
                self.ai_names.append(name)
                self.ai_paths.append(os.path.join(path,pic))
        self.ai_rect = []
        for (i, valu) in enumerate(self.ai_names):
            rect = pg.Rect(10 +  name_width, 170 + i * name_height, name_width, name_height)
            self.ai_rect.append(rect)

    def get_event(self, event):
        super().get_event(event)
        if event.type == pg.MOUSEBUTTONDOWN:
            for (i,rect) in enumerate(self.ai_rect):
                if rect.collidepoint(event.pos):
                    if self.ai_names[i] == 'airandom':
                        self.ais.append(airandom.AI(self.persist['manager']))
                    elif self.ai_names[i] == 'aiplayer':
                        self.ais.append(aiplayer.AI(self.persist['manager']))
                    elif self.ai_names[i] == 'aicircle':
                        self.ais.append(aicircle.AI(self.persist['manager']))
                    elif self.ai_names[i] == 'aichose':
                        self.ais.append(aichose.AI(self.persist['manager']))
                    self.phase += 1

    def update(self, time_passed):
        super().update(time_passed)
        if self.phase == 2:
            self.next = c.SCENE_BATTLE
            self.done = True

    def cleanup(self):
        self.done = False
        self.persist['ais'] = self.ais
        return self.persist

    def startup(self, current_time, persistant):
        super().startup(current_time, persistant)
        # self.manager = persistant['manager']
        self.phase = 0
        self.ais = []

    def draw(self, screen):
        screen.fill(c.GREEN)
        tools.draw_text(screen, 40, (324,40), "AI选择", c.GOLD)
        if self.phase != 2:
            string = "请为%s选择AI:" % c.INIT_BOT[self.phase]['name']
            tools.draw_text(screen, 30, (150,120), string, c.INIT_BOT[self.phase]['color'])

        for (i,name) in enumerate(self.ai_names):
            pg.draw.rect(screen, c.RED, self.ai_rect[i], 1)
            tools.draw_text(screen, name_height, self.ai_rect[i].topleft, name, c.PURPLE)

        self.btn_group.draw(screen)

    def back(self):
        self.next = c.SCENE_START
        self.done = True


import matplotlib.pyplot as plt
import numpy as np
import csv
from os import listdir
from os.path import isfile, join
from trajectory_vars import *

#plt.style.use('dark_background')

V1, V2, V3 = 'vel: 1', 'vel: 2', 'vel: 3'

COLORS = {
    'tree': '#788e55',
    V1: '#531212',
    V2: '#62a8dd',
    V3: '#cfa072',
}

TOPIC = '/mavros/local_position/pose'

def read_arrays(marker, topic=TOPIC):
    path = 'npys/'
    files = get_files(marker, path)
    return [{'points': np.load(join(path, f)), 'vel': get_vel(f)} for f in files]

def get_vel(fname):
    if 'vel-10' in fname:
        return V1
    elif 'vel-20' in fname:
        return V2
    elif 'vel-30' in fname:
        return V3

def get_files(marker, path):
    marker = str(marker)
    files = [f for f in listdir(path) if isfile(join(path, f))
             and f[:len(marker)] == marker]
    return files

def read_world(fname):
    with open(join('csv/', fname + '.csv')) as world_file:
        reader = csv.DictReader(world_file, delimiter=',')
        rows = []
        for row in reader:
            rows.append(row)
        return rows

def plot_trajectory(marker, fname_world):

    # Obstacles
    obstacles = read_world(fname_world)
    fig, ax = plt.subplots(figsize=(8, 8))
    xs, ys = [], []
    c = 0.65

    for obstacle in obstacles:
        xs.append(float(obstacle[X]))
        ys.append(- float(obstacle[Y]))

    ax.plot(ys, xs, 'o', color=COLORS['tree'], markersize=10, label='tree')
    for size in np.linspace(0, 1, 21):
        ax.plot(ys, xs, 'o', color=COLORS['tree'], markersize= c * size * float(obstacle[SIZE]), alpha=max(0.1, 0.8-size))

    # Trajectories
    trajectories = read_arrays(marker)
    xs, ys = [], []
    for trajectory in trajectories:
        xs = trajectory['points'][:, 0]
        ys = -1. * trajectory['points'][:, 1]
        label = trajectory['vel']
        ax.plot(ys, xs, color=COLORS[label], label=label, alpha=1)
    ax.legend()
    ax.set_xlim(-20, 40)
    ax.set_ylim(10, 70)
    fig.tight_layout(rect=[0, 0.03, 1, 0.95])
    plt.savefig('figs/' + str(marker) + '-trajectories-' + fname_world + '.pdf')

plot_trajectory(0, 'circuit-trees')


import numpy as np
import matplotlib.pyplot as plt
import sys
sys.path.append('/Users/haruto/Desktop/mainwork/codes/MMdialogueSystem')
import pandas as pd
import pickle
import os
import itertools
from sklearn import preprocessing

# ソフトマックス関数
# coefは推定値の振れ幅を調整するためのもの．（デフォルトは1）
def softmax(a, coef=1):
    # 一番大きい値を取得
    c = np.max(a)
    # 各要素から一番大きな値を引く（オーバーフロー対策）
    exp_a = np.exp(coef * (a - c))
    sum_exp_a = np.sum(exp_a)
    # 要素の値/全体の要素の合計
    y = exp_a / sum_exp_a
    return y

# ファイルが既に存在する場合，代わりの名前を振ってあげる．
def search_and_rename_filename(oldpath):
    if os.path.exists(oldpath):
        print('file "{}" already exists.'.format(oldpath))
        #dirpath:ディレクトリのパス, filename:対象のファイルまたはディレクトリ
        #name:対象のファイルまたはディレクトリ（拡張子なし）, ext:拡張子
        dirpath, filename = os.path.split(oldpath)
        name, ext = os.path.splitext(filename)

        for i in itertools.count(1):
            newname = '{}_{}{}'.format(name, i, ext)
            newpath = os.path.join(dirpath, newname)
            if not os.path.exists(newpath):
                return newpath
            else:
                print('file "{}" already exists.'.format(newpath))
    else:
        return oldpath

class ELAgent():

    def __init__(self, epsilon):
        self.Q = {}
        self.epsilon = epsilon
        self.reward_log = []
        self.dialogue_log = []
        self.max_n_exchg = 10

    # epsilon以下でランダムな行動，それ以外はQに従った行動
    # softmax=Trueで確率的に選択するようにできます
    def policy(self, s, actions, selection='argmax'):

        if selection == 'argmax':
            if np.random.random() < self.epsilon:
                return np.random.randint(len(actions))
            else:
                if s in self.Q and sum(self.Q[s]) != 0:
                    return np.argmax(self.Q[s])
                else:
                    return np.random.randint(len(actions))
        elif selection == 'softmax':
            if np.random.random() < self.epsilon:
                return np.random.randint(len(actions))
            else:
                if s in self.Q and sum(self.Q[s]) != 0:
                    return np.argmax(softmax(preprocessing.minmax_scale(self.Q[s])))
                else:
                    return np.random.randint(len(actions))
        else:
            print('invalid "selection"')
            exit(0)

    def init_log(self):
        self.reward_log = []
        self.dialogue_log = []

    def append_log_reward(self, reward):
        self.reward_log.append(reward)

    def append_log_dialogue(self, exchgID, state, action, theme, impression, s_utte, u_utte):
        self.dialogue_log.append([exchgID+'_S', state, action, theme, '-', s_utte])
        self.dialogue_log.append([exchgID+'_U', '-', '-', '-', impression, u_utte])

    def show_reward_log(self, interval=50, episode=-1, filename='sample.png'):
        if episode > 0:
            rewards = self.reward_log[-interval:]
            mean = np.round(np.mean(rewards), 3)
            std = np.round(np.std(rewards), 3)
            print("At Episode {} average reward is {} (+/-{}).".format(
                   episode, mean, std))
        else:
            indices = list(range(0, len(self.reward_log), interval))
            means = []
            stds = []
            for i in indices:
                rewards = self.reward_log[i:(i + interval)]
                means.append(np.mean(rewards))
                stds.append(np.std(rewards))
            means = np.array(means)
            stds = np.array(stds)
            plt.figure()
            plt.title("Reward History")
            plt.grid()
            plt.fill_between(indices, means - stds, means + stds,
                             alpha=0.1, color="g")
            plt.plot(indices, means, "o-", color="g",
                     label="Rewards for each {} episode".format(interval))
            plt.legend(loc="best")
            plt.savefig(filename)
            #plt.show()

    def write_dialogue_log(self, filename):
        df = pd.DataFrame(data=self.dialogue_log, columns=['exchgID', 'state', 'action', 'theme', 'UI', 'utterance'])
        filename_new = search_and_rename_filename(filename)
        df.to_csv(filename_new, index=None)
        print('finished making file "{}".'.format(filename_new))

    def saveR(self, filename):
        np.save(filename, np.array(self.reward_log))

    def saveQ(self, table, filename):
        with open(filename, mode='wb') as f:
            pickle.dump(dict(table), f)


from django.urls import path

from .views import (
	AccountSignUpView,
	AccountSignInView,
	GuestSignUpView,
)

urlpatterns = [
	path('/sign-up', AccountSignUpView.as_view()),
	path('/sign-in', AccountSignInView.as_view()),
	path('/guest/sign-up', GuestSignUpView.as_view()),
]


import requests
from pyquery import PyQuery as pq
import time

base_url = 'https://tieba.baidu.com/p/{}?see_lz={}'

class BDZD(object):
    def __init__(self,topicId,seeLz=1):
        self.topicId = topicId
        self.seeLz = seeLz
        self.url = base_url.format(self.topicId,self.seeLz)
        self.currentPageNum = 1
        self.currentPageContent = ''
        self.totalPage = 1

    def geneCurrentPage(self):
        def formatStr(src,width=40):
            index = 1
            while index*width<len(src):
                src = src[:index*width] + '\n' + src[index*width:]
                index += 1
            return src

        url = self.url + "&pn=" + str(self.currentPageNum)
        r = requests.get(url)
        divs = pq(r.text).find('div.d_post_content.j_d_post_content')
        index = 1
        self.currentPageContent = ''
        self.currentPageContent += '\n**********************第{}页**********************\n'.format(self.currentPageNum)
        for i in divs.items():
            self.currentPageContent += '\n======================第{}楼======================\n'.format(index)
            self.currentPageContent += formatStr(i.text())
            index += 1

    def saveToFile(self,fileName):
        with open(fileName,'a') as f:
            print('正在保存第{}页'.format(self.currentPageNum))
            f.write(self.currentPageContent)

    def initPageCount(self):
        url = self.url + "&pn=1"
        r = requests.get(url)
        count = pq(r.text).find('div.l_thread_info li.l_reply_num span.red').eq(1).text()
        self.totalPage = count

    def main(self):
        self.initPageCount()
        print('总页数为：{}'.format(self.totalPage))
        for i in range(int(self.totalPage)):
            self.geneCurrentPage()
            self.saveToFile(str(self.topicId)+'_' + time.strftime('%Y-%m-%d')+ '.txt')
            self.currentPageNum += 1

if __name__ == '__main__':
    bdzd = BDZD(3138733512)
    bdzd.main()


import unittest
from tests import PluginTest
from plugins.calories import Calories
from Jarvis import Jarvis

class caloriesTest(PluginTest):
    """
        This class is testing the calories.py plugin
    """

    def setUp(self):
        self.test = self.load_plugin(Calories)

    def testCalories1(self):
        strings = {}
        strings[0] = "f"
        strings[1] = 35
        strings[2] = 165
        strings[3] = 70
        strings[4] = 4
        result = self.test.calories(Jarvis, strings)
        self.assertEqual(result, 2720.73)

    def testCalories2(self):
        strings = {}
        strings[0] = "m"
        strings[1] = 20
        strings[2] = 184
        strings[3] = 87
        strings[4] = 4
        result = self.test.calories(Jarvis, strings)
        self.assertEqual(result, 3753.75)

    def testCalories3(self):
        strings = {}
        strings[0] = "I"
        strings[1] = 20
        strings[2] = 20
        strings[3] = 30
        strings[4] = 20
        result = self.test.calories(Jarvis, strings)
        self.assertEqual(result, None)

    def testExercise_level1(self):
        level = 3
        result = self.test.exercise_level(level)
        self.assertEqual(result, 1.6)

    def testExercise_level2(self):
        level = 10
        result = self.test.exercise_level(level)
        self.assertEqual(result, 1)


import os
import asyncio
import json
import aiohttp
from understat import Understat

MATCHES_DIR = 'matches'
MATCHES_SHOTS_DIR = 'matches_shots'
os.makedirs(MATCHES_SHOTS_DIR, exist_ok=True)

matches_shots = None
async def main(match_id):
    async with aiohttp.ClientSession() as session:
        understat = Understat(session)

        try:
            match_shots = await understat.get_match_shots(
                match_id,
            )
        except:
            print('Match {} could not be retrieved!'.format(match_id))
            match_shots = None

        matches_shots.append(match_shots)

loop = asyncio.get_event_loop()

leagues = ['epl', 'la_liga', 'bundesliga', 'serie_a', 'ligue_1']
years = ['2014', '2015', '2016', '2017', '2018', '2019']

for league in leagues:
    for year in years:
        print('Process {} {}'.format(league, year))
        matches = json.load(open(os.path.join(MATCHES_DIR, 'matches_{}_{}.json'.format(league, year)), 'r'))
        matches_shots = []
        for idx, match in enumerate(matches):
            if (idx + 1) % 10 == 0:
                print('Process match {}/{}'.format(idx + 1, len(matches)))
            loop.run_until_complete(main(int(match['id'])))

        json.dump(matches_shots, open(os.path.join(MATCHES_SHOTS_DIR, 'matches_shots_{}_{}.json'.format(league, year)), 'w'))


from netifaces import AF_INET
import netifaces as nf
import datetime
import sys
import os
import subprocess
import threading

#create a place to send certain command output

FNULL=open(os.devnull,'w')

#make sure all arguments are provided, if not then exit with this message

if len(sys.argv)!=4:

	print("Usage: python3 pyconnaissance.py <interface> <log file> <desired run time(seconds)>")

	exit()

#write command output to log file

def logcmdout(args):

	subprocess.run(args,stdout=f)

#get host address on current interface

def get_local_addr():

	local_addr=nf.ifaddresses(sys.argv[1])[AF_INET][0]['addr']

	return local_addr

#get subnet to be used by nmap

def get_ip_range():

	t=0

	i=0

	x=' '

	addr=get_local_addr()

	addr_len=len(addr)

	while x!='.':

		i+=1

		t=addr_len-i

		x=addr[t]

	addr=addr[:t]

	addr=addr+".0/24"

	f.write("IP range to scan: "+addr+"\n")

	return addr

#get the router ip address

def get_default_gateway():

	t=0

	i=0

	x=' '

	addr=get_local_addr()

	addr_len=len(addr)

	while x!='.':

		i+=1

		t=addr_len-i

		x=addr[t]

	addr=addr[:t]

	addr=addr+".1"

	f.write("Default Gateway: "+addr+"\n")

	return addr

#use nmap to scan all hosts on the network

def netmap(range_ip):

	os.system("nmap -T4 -O -sV -oX - "+range_ip+" > "+sys.argv[2]+".xml")

#convert nmap's xml output to html

def xml_to_html():

	os.system("xsltproc "+sys.argv[2]+".xml > "+sys.argv[2]+".html")

	os.system("rm "+sys.argv[2]+".xml")

#start arpspoof to prepare for network sniffing

def arpspoof():

	time=str(int(sys.argv[3])+10)

	f.write("Started arpspoof on "+sys.argv[1]+" - target is "+default_gateway+"\n")

	arp_proc=subprocess.Popen(['timeout',time,'arpspoof','-i',sys.argv[1],default_gateway],stdout=FNULL,stderr=FNULL)

#start tcpdump

def tcpdump():

	t=subprocess.run(["tcpdump","-i",sys.argv[1],"-W","1","-vv","-n","-w",sys.argv[2]+".pcap","-G",sys.argv[3]],stdout=FNULL,stderr=FNULL)

#sniff the network

def netsniff(router_ip):

	iptables_flush()

	arpthread=threading.Thread(target=arpspoof)

	arpthread.start()

	tcpdump()

	arpthread.join()

	iptables_flush()

#solution to some arpspoof issues

def iptables_flush():

	f.write("Flushing iptables....\n")

	logcmdout(['iptables','--flush'])

	logcmdout(['iptables','--table','nat','--flush'])

	logcmdout(['iptables','--delete-chain'])

	logcmdout(['iptables','--table','nat','--delete-chain'])

#main code

#get start time

start_time = datetime.datetime.now()

#create log file

f=open(sys.argv[2], "w+")

#write time into log file

f.write("Started at "+str(start_time)+"\n")

#write host address into log file

f.write("Host Address: "+get_local_addr()+"\n")

#set value to the subnet for nmap

ip_range=get_ip_range()

#set value to router ip

default_gateway=get_default_gateway()

#run nmap

netmap(ip_range)

#convert xml output of nmap to html

xml_to_html()

#sniff the network

netsniff(default_gateway)

#close main log

f.close()


from config import load_config
from workers import use_worker
import logging
import os
from pathlib import Path
import posixpath
import datetime
import json

if __name__ == '__main__':
    build_version = '1.0.0'
    file_log_level = os.getenv("file_log_level", "INFO")
    # log folder under /app. Ex: /app/log/logfile
    log_path = Path('./log')
    log_path.mkdir(parents=True, exist_ok=True)
    logger = logging.getLogger('__name__')
    logger.setLevel(file_log_level)

    logger.debug(f"[start_worker] file log level: {file_log_level}")
    log_file = 'log'
    file_log = {
        "launch_info": dict(),
        "time_start": str(datetime.datetime.utcnow())
    }

    logger_fh = logging.FileHandler(posixpath.join(log_path, log_file))
    formatter_fh = logging.Formatter('%(message)s')
    logger_fh.setFormatter(formatter_fh)
    logger.addHandler(logger_fh)

    worker_type, worker_config, file_log_config = load_config(file_log)
    file_log["launch_info"]["build_version"] = build_version
    file_log["launch_info"]["video_download_dir"] = worker_config.get("video_download_dir", None)

    file_log["launch_info"]["clean_folder"] = worker_config.get("clean_folder", None)
    file_log["launch_info"]["smart_download"] = worker_config.get("smart_download", None)
    file_log["launch_info"]["log_file"] = worker_config.get("log_file", None)

    logger.info(json.dumps(file_log_config))
    worker = use_worker(worker_type)(**worker_config)
    worker.run()


"""Cyber_Hacking_Breaches URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/2.0/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.conf.urls import url
from django.contrib import admin
from Cyber_Admins import views as admin_view
from app import views as user_view

urlpatterns = [
    url('admin/', admin.site.urls),

    url(r'^$',user_view.user_login, name='user_login'),
    url(r'^user_register/$',user_view.user_register,name='user_register'),
    url(r'^user_adddata/$',user_view.user_adddata,name='user_adddata'),
    url(r'^user_page/$',user_view.user_page,name='user_page'),
    url(r'^malware/$',user_view.malware,name='malware'),
    url(r'^unmalware/$',user_view.unmalware,name='unmalware'),
    url(r'^breaches_analysis/$',user_view.breaches_analysis,name='breaches_analysis'),
    url(r'chart_page/(?P<chart_type>\w+)', user_view.chart_page, name="chart_page"),

    url(r'^admin_login/$',admin_view.admin_login, name='admin_login'),
    url(r'^user_details/$',admin_view.user_details, name='user_details'),
    url(r'^admin_analysis/$',admin_view.admin_analysis, name='admin_analysis'),
    url(r'^achart_page/(?P<achart_type>\w+)', admin_view.achart_page, name="achart_page"),

]


import modelnet40

#"""
#HW1 Part II
#"""
#
import os, os.path
import pprint
import glob
import random
from keras.applications.resnet50 import preprocess_input, decode_predictions
from keras.preprocessing import image
import numpy as np
import numpy
#"""
#from keras.applications.resnet50 import ResNet50
#from keras.preprocessing import image
#from keras.applications.resnet50 import preprocess_input, decode_predictions
#"""
from keras import applications
from keras.preprocessing.image import ImageDataGenerator
from keras import optimizers
from keras.models import Sequential
from keras.layers import Dropout, Flatten, Dense,Input,concatenate,Maximum,Conv2D
from keras.models import Model
from keras import models
from keras.layers.normalization import BatchNormalization
#DEFAULT_SRCDIR = 'modelnet40'
#DEFAULT_TARGET_SIZE = (224, 224)        # Input size for ResNet-50
#nclasses = 40
#nviews = 12
#
#def subdirs(dirname):
#    return [x for x in os.listdir(dirname) if os.path.isdir(os.path.join(dirname, x))]
#
#def read_image(filename, target_size, preprocess=None):
##    print('read_image:', filename)
#    x = image.load_img(filename, target_size=target_size)
#    x = image.img_to_array(x)
#    x = np.expand_dims(x, axis=0)
#    if preprocess is not None:
#        x = preprocess(x)
#    #print('read_image, shape:', x.shape)
#    return x
#
#def modelnet40_filenames(subset, src_dir=DEFAULT_SRCDIR):
#    """
#    List of models for ModelNet-40.
#
#    Each model is a pair (class_index, filename_L).
#    Here filename_L is a 12 length list of image filenames of views of the model.
#    """
#    src_dir = os.path.join(src_dir, 'classes')
#    classes = sorted(subdirs(src_dir))
#    ans = []
#    for (icls, cls) in enumerate(classes):
#        subset_dir = os.path.join(src_dir, cls, subset)
#        model_dirs = subdirs(subset_dir)
#        for model_dir in model_dirs:
#            filenames = glob.glob(os.path.join(src_dir, cls, subset, model_dir, '*.png'))
#            ans.append((icls, filenames))
#    return ans
#
#def modelnet40_generator(subset, src_dir=DEFAULT_SRCDIR, single=True, target_size=DEFAULT_TARGET_SIZE, repeats=None, shuffle=True, verbose=0, frac=1.0, class_array=True, preprocess=preprocess_input):
#    """
#    A generator that returns images and classes from ModelNet-40 in size one batches.
#
#    Returns (g, dataset_size), where g is the generator and dataset_size is the number of elements in the dataset.
#
#    The generator yields by default an infinite number of elements which each have the form (x, y), where x is
#    an input for supervised training and y is an output. If single is True (single view mode) then x is a 4D numpy
#    array with shape 1 x h x w x 3, representing an input image, and y is a 2D tensor of shape 1 x nclasses, where
#    nclasses is the number of classes in ModelNet-40 (defined as the global nclasses, equal to 40). If single is False
#    (multiple view mode) then x is a list of arrays representing different views of the same model: the list has length
#    nviews (defined as the global nviews, equal to 12): each view is an numpy array of an image with shape 1 x h x w x 3.
#
#    Arguments:
#    - subset:       Either 'train' or 'test'
#    - src_dir:      The ModelNet-40 directory.
#    - single:       If true, return one image and class at a time in the format (img, cls).
#                    If false, return a list of (img, cls) for all (12) views.
#    - repeats:      Number of times to repeat the dataset. If None, repeat forever.
#    - shuffle:      If true, randomly shuffle the dataset.
#    - verbose:      Print information about loading: verbose=0: no info, 1 is some info, verbose=2 is more info.
#    - frac:         Fraction of dataset to load (use frac < 1.0 for quick tests).
#    - class_array:  If true, return class as a 1-hot vector array.
#    - preprocess:   Preprocessing function from a Keras model to be called on each image (numpy array) after being read.
#                    (or None, the default, for no preprocessing).
#    """
#    viewL = modelnet40_filenames(subset, src_dir)
#    def generator_func():
#        repeat = 0
#        while repeats is None or repeat < repeats:
#            if shuffle:
#                random.shuffle(viewL)
#            for (i, view) in enumerate(viewL[:int(len(viewL)*frac)]):
#                if verbose == 1 and i % 100 == 0:
#                    print('Loading %s data: %.1f%%' % (subset, i*100.0/(len(viewL)*frac)))
#                (cls, view) = view
#                if verbose == 2:
#                    print('Loading data point %d, cls = %d' % (i, cls))
#                if class_array:
#                    cls_array = numpy.zeros((1, nclasses), 'float32')
#                    cls_array[0, cls] = 1.0
#                    cls = cls_array
#                if single:
#                    filename = random.choice(view)
#                    yield (read_image(filename, target_size, preprocess), cls)
#                else:
#                    yield ([read_image(view_elem, target_size, preprocess) for view_elem in view], cls)
#            repeat += 1
#    return (generator_func(), len(viewL))
def batchGenerator( SingleGenerator, batchSize, dataSetSize,single=True,augment=True):
	def nestedGen():
		while True:
			for i in range(batchSize):
				(a,b)=SingleGenerator.__next__()
#				print("len a is",len(a))
#				print(a[0])
				if(augment==True and single==True):
					a[0]=image.random_rotation(a[0],45, row_axis=0, col_axis=1, channel_axis=2,fill_mode='nearest', cval=0.)
				if(augment==True and single==False):
					for j in range(len(a)):
						a[j][0]=image.random_rotation(a[j][0],45, row_axis=0, col_axis=1, channel_axis=2,fill_mode='nearest', cval=0.)

					#random horizontal flip along columns
				if(augment==True and single==True):
					rnd=np.random.uniform(0,1)
					if(rnd<0.5):
						a=image.flip_axis(a, 2)
				if(augment==True and single==False):
					for j in range(len(a)):
						rnd=np.random.uniform(0,1)
						if(rnd<0.5):
							a[j]=image.flip_axis(a[j], 2)
				if(i==0):
					(array_a,array_b)=(a,b)
				else:
	#				print("hi")
					if(single==True):
						array_a=np.append(array_a,a,axis=0)
						array_b=np.append(array_b,b,axis=0)
					else:
#						print(len(a))
						for j in range(len(a)):
							array_a[j]=np.append(array_a[j],a[j],axis=0)
						array_b=np.append(array_b,b,axis=0)
#					print(a.shape, b.shape)
	#						print(array_a[j].shape)
			yield (array_a,array_b)
	return (nestedGen(),dataSetSize )
STOP_LAYER=20
if __name__ == '__main__':

    (g, dataset_size) = modelnet40.modelnet40_generator('test')
    print('Loading first element from dataset')
    (x1, y1) = g.__next__()                             # Python 3 syntax. Use .next() instead for Python 2.
    print(x1, y1)
    print('Loading second element from dataset')
    (x2, y2) = g.__next__()                             # Python 3 syntax. Use .next() instead for Python 2.
    print(x2, y2)
    print('Done loading')
    print(x2.shape)
    (Tgen,Tsize)=batchGenerator(g,5,dataset_size)
    print(Tgen.__next__()[0].shape )

####################################################
    sharedResnet = applications.resnet50.ResNet50(include_top=False,input_shape=x2.shape[1:])
    sharedResnet=models.Model(sharedResnet.input, sharedResnet.layers[STOP_LAYER].output)
    Input_instances=[Input(shape=x2.shape[1:]) for i in range(12)]
    resnet_instances=[sharedResnet(Input_instances[i]) for i in range(12)]
#    merged_vector = concatenate(resnet_instances, axis=-1)
    base_model=Maximum()(resnet_instances)
    x = base_model
    x=Conv2D(32, (3, 3), padding='same', activation='relu')(x)
    x=BatchNormalization()(x)
    x=Conv2D(32, (3, 3), padding='same', activation='relu')(x)
    x=BatchNormalization()(x)
    x= Flatten(input_shape=x.shape[1:])(x)
    predictions= Dense(40, activation='softmax')(x)
    whole_model = Model(Input_instances, outputs=predictions)
#    print (resnet.layers)
    p=int(0.7*len(whole_model.layers))
    print(len(whole_model.layers), p)
    for layer in whole_model.layers[:p]:
    	layer.trainable = False

    nb_train_samples = 2000
    nb_validation_samples = 800
    epochs = 50
    batchSize=5
    augment=True
    single=False
    whole_model.compile(loss='categorical_crossentropy',optimizer=optimizers.SGD(lr=1e-4, momentum=0.9), metrics=['accuracy'])
    (train_generator, tSize)= modelnet40.modelnet40_generator('train', src_dir=modelnet40.DEFAULT_SRCDIR, single=single, target_size=modelnet40.DEFAULT_TARGET_SIZE, repeats=None, shuffle=True, verbose=0, frac=1.0, class_array=True, preprocess=preprocess_input)
    (validation_generator,vSize)=modelnet40.modelnet40_generator('test', src_dir=modelnet40.DEFAULT_SRCDIR, single=single, target_size=modelnet40.DEFAULT_TARGET_SIZE, repeats=None, shuffle=True,  verbose=0, frac=1.0, class_array=True, preprocess=preprocess_input)
    (batchTrainGen,btSize)=batchGenerator(train_generator,batchSize,tSize,single=single,augment=augment)
    (batchValidGen,bvSize)=batchGenerator(validation_generator,batchSize,vSize,single=single,augment=augment)
    whole_model.fit_generator(batchTrainGen, samples_per_epoch=nb_train_samples,epochs=epochs,validation_data=batchValidGen,nb_val_samples=nb_validation_samples)


import datetime
import os
import mundi
import pandas as pd
from subprocess import run
from pydemic.diseases import disease
from pathlib import Path
from matplotlib import pyplot as plt
# from mundi.plugins.epidemic import covid19

covid19 = disease("covid-19")
PATH = Path(__file__).parent / "data"

def plot_region(path: Path, region: mundi.Region, ext='png'):
    """
    Plot state.
    """
    df = pd.read_csv(path / 'epicurve.csv').iloc[1:]
    plot_data(path, df, region, ext)
    df.index.name = 'day'
    return df

def plot_data(path, data: pd.DataFrame, region, ext='png', future=60, past=120):
    data = data.iloc[-(past + future):].copy().reset_index()
    data.index -= past
    today = datetime.datetime.now().date()

    def common(name):
        plt.legend()
        plt.tight_layout()
        _y0, y1 = plt.ylim()
        plt.plot([0, 0], [0, y1], 'k--', lw=2)
        plt.ylim(0, y1)
        plt.xlim(data.index[0], data.index[-1])
        plt.grid(True)
        # plt.show()
        plt.savefig(path / name)
        plt.clf()

    deaths = data['new_deaths']
    deaths.rolling(14, 1, center=True, win_type='triang').mean().plot(label='média móvel')
    deaths.plot(label='mortes/dia')
    plt.xlabel(f'dias (a partir de {today})')
    plt.ylabel('mortes')
    plt.title(f'Projeção de óbitos por Covid-19 ({region.name})')
    common(f'obitos.{ext}')

    cases = data['new_cases']
    cases.rolling(14, 1, center=True, win_type='triang').mean().plot(label='média móvel')
    cases.plot(label='casos/dia')
    plt.xlabel(f'dias (a partir de {today})')
    plt.ylabel('casos')
    plt.title(f'Projeção de casos por Covid-19 ({region.name})')
    common(f'casos.{ext}')

    icu = data['C']
    icu.rolling(14, 1, center=True, win_type='triang').mean().plot(label='média móvel')
    icu.plot(label='leitos UTI')
    plt.xlabel(f'dias (a partir de {today})')
    plt.ylabel('leitos ocupados')
    plt.title(f'Projeção de pressão hospitalar por Covid-19 ({region.name})')
    common(f'criticos.{ext}')

def plot_all(path: Path = PATH):
    """
    Prepare all states data
    """
    states = mundi.regions(type="state", country="BR")
    data = []
    for r in sorted(states):
        print(f"\nprocessing {r}")
        data.append(plot_region(PATH / r.id, r).reset_index())

    br = path / "BR"
    br.mkdir(exist_ok=True)
    df = pd.concat(data).groupby('day').sum()
    plot_data(br, df, mundi.region('BR'))
    df.to_csv(br / 'epicurve.csv')

if __name__ == "__main__":
    import typer

    typer.run(plot_all)


import pytest
import numpy
from numpy.testing import assert_allclose
from cytoolz import concat

from ....linear.avgtron import AveragedPerceptron

@pytest.fixture
def templates():
    return (
        (10,),
        (2,),
        (10, 2),
        (689,)
    )

@pytest.fixture
def atoms(templates):
    atoms = numpy.zeros((max(concat(templates)),), dtype='uint64')
    atoms[10] = 100
    atoms[2] = 50009
    return atoms

@pytest.fixture
def nr_class():
    return 6

@pytest.fixture
def model(templates, nr_class):
    return AveragedPerceptron(templates, nr_out=nr_class)

def test_init(templates, model):
    assert model.nr_feat == len(templates) + 1

@pytest.mark.xfail
def test_call(model, atoms):
    scores = model(atoms)
    assert isinstance(scores, numpy.ndarray)
    assert scores.shape == (model.nr_out,)
    assert not numpy.isnan(scores.sum())

@pytest.mark.skip
def test_predict_batch(model, atoms):
    pass

@pytest.mark.xfail
def test_update_scores_match_call(model, atoms):
    atoms = numpy.expand_dims(atoms, 0)
    scores_via_update, finish_update = model.begin_update(atoms)
    scores_via_call = model(atoms[0])
    assert_allclose(scores_via_update[0], scores_via_call)

@pytest.mark.xfail
def test_finish_update_executes(model, atoms):
    atoms = numpy.expand_dims(atoms, 0)
    scores, finish_update = model.begin_update(atoms)
    assert scores.shape == (1, model.nr_out)
    labels = numpy.zeros(scores.shape[:1], dtype='uint64')
    labels[0] = model.nr_out-1
    finish_update(labels)


#  coding: utf-8
import socketserver
import signal
import sys
import os

# Copyright 2019 Carlo Oliva
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
# Furthermore it is derived from the Python documentation examples thus
# some of the code is Copyright © 2001-2013 Python Software
# Foundation; All Rights Reserved
#
# http://docs.python.org/2/library/socketserver.html
#

# --------- Globally Defined Variables ---------- #

"""
Status codes taken from https://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html

Carriage Return and New Line taken from class notes
    CMPUT 404 Abram Hindle January 18, 2019
"""

OK = "HTTP/1.1 200 OK\n"
ERROR404 = "HTTP/1.1 404 Not Found\n"
ERROR405 = "HTTP/1.1 405 Method Not Found\n"
HTML = "Content-Type: text/html\r\n\r\n"
CSS = "Content-Type: text/css\r\n\r\n"

# ----------------- Classes --------------------- #

class MyWebServer(socketserver.BaseRequestHandler):

    def handle(self):
        self.data = self.request.recv(1024).strip().decode("utf-8")
        if self.data.startswith('GET'):
            print("Request: " + self.data.split("\r\n")[0])
            url_file, code = decompose_request(self.data)
            print("Sending Request Type: " + code + "\n")
            if code == '200':
                if url_file.endswith("index.html"):
                    sendall_html = generate_sendall(url_file, 'html')
                    self.request.sendall(sendall_html.encode())
                if url_file.endswith('.css'):
                    sendall_css = generate_sendall(url_file, 'css')
                    self.request.sendall(sendall_css.encode())
            elif code == '301':
                sendall_redirect = generate_sendall(url_file, 'redirect')
                self.request.sendall(sendall_redirect.encode())
            else:
                self.request.sendall(ERROR404.encode())

        else:
            # Return status code 405 Method not allowed for (POST/PUT/DELETE)
            self.request.sendall(ERROR405.encode())

# ------------ User Defined Functions ------------#

def decompose_request(data):
    """
    Takes the url of the get request and parse through it to generate the correct file path.
    :param data: the url
    :return: file path
    """
    url = data.split("\r\n")[0].split(" ")[1]
    prefix = 'www'
    delimiter = '/'
    spl = url.split("/")
    length = len(spl)
    if spl[length - 1] == "":
        return existing_file(spl, delimiter, prefix, 1)
    elif not spl[length - 1].endswith(".html") and not spl[length - 1].endswith(".css"):
        return existing_file(spl, delimiter, prefix, 2)
    elif spl[length - 1].endswith(".css"):  # Handle serving css
        return existing_file(spl, delimiter, prefix, 3)

    return prefix + delimiter.join(spl), '200'

def existing_file(spl, delimiter, prefix, if_type):
    """
    Checks if the file requested exists, and whether or not to send a 200, 301 or 404

    :param spl: The url split into a list
    :param delimiter: '/'
    :param prefix: www
    :param if_type: Integer
    :return: data, request type
    """
    if if_type in (1, 2):
        file_exists = os.path.isfile('www/{}/index.html'.format("/".join(spl)))
        if file_exists:
            if if_type == 1:
                return 'index.html', '301'
            else:
                spl.append("index.html")
                return delimiter.join(spl), '301'
        else:
            return "", '404'
    if if_type == 3:
        file_exists = os.path.isfile('www/{}'.format("/".join(spl)))
        if file_exists:
            return prefix + delimiter + "/".join(spl)[1:], '200'
        else:
            return "", '404'

def generate_sendall(file, req_type):
    """
    Adds the correct heading to the HTML/CSS files found in the www folder

    :param file: the html/css file to be opened and read
    :param req_type: either html/css/redirect
    :return: A complete string including the correct headings
    """
    if req_type == 'html':
        sendall = OK + HTML
    elif req_type == 'redirect':
        sendall = "HTTP/1.1 301 Moved Permanently\r\nLocation: {}\n".format(file)
        return sendall
    else:
        sendall = OK + CSS
    with open(file, 'r', encoding='utf-8') as infile:
        for line in infile:
            sendall += line
    return sendall

def signal_handler(sig, frame):
    """
    Checks if control+c is pressed to stop the web server for a more graceful exit than displaying the exception
    messages.

    Taken from https://stackoverflow.com/questions/1112343/how-do-i-capture-sigint-in-python on January 14/19
    Author: https://stackoverflow.com/users/18528/matt-j
    :param sig:
    :param frame:
    :return: None
    """
    print('\nStopping server!')
    sys.exit(0)

if __name__ == "__main__":
    HOST, PORT = "localhost", 8080

    socketserver.TCPServer.allow_reuse_address = True
    # Create the server, binding to localhost on port 8080
    server = socketserver.TCPServer((HOST, PORT), MyWebServer)

    # Print starting state
    print("Server started on http://127.0.0.1:8080/")
    print("To exit press Ctrl-c")
    signal.signal(signal.SIGINT, signal_handler)

    # Activate the server; this will keep running until you
    # interrupt the program with Ctrl-C
    server.serve_forever()

    signal.pause()


import sys
kb = sys.stdin
code = kb.readline().strip()
decrypt1 = 10000 + int(code[3]+code[10]+code[17]+code[24]+code[31]) + int(code[7]+code[12]+code[17]+code[22]+code[27])
decrypt1 = str(decrypt1)
decrypt2 = int(decrypt1[-2]+decrypt1[-3]+decrypt1[-4])
decrypt2 = str(decrypt2)
if decrypt1[-2] == "0":
    decrypt2 = "0" + decrypt2
    if decrypt1[-3] == "0":
        decrypt2 = "0" + decrypt2
        if decrypt1[-4] == "0":
            decrypt2 = "0" + decrypt2
result = decrypt2
decrypt3 = int(decrypt2[0]) + int(decrypt2[1]) + int(decrypt2[2])
decrypt3 = str(decrypt3)
decrypt4 = int(decrypt3[-1]) + 1
alphabet = "ABCDEFGHIJ"
print(result + alphabet[decrypt4-1])


import json
from app.sales_data.models.sales_data import SalesData
from ..conftest import populate_users
import io
import os
import csv

def test_sales_data_upload(auth, client, app):
    with app.app_context():
        populate_users(app)

        # get auth_token
        res = auth.login()
        token = json.loads(res.data.decode())

        headers = {
            "Authorization": "Bearer {}".format(token["access_token"])
        }

        file_path = os.path.dirname(os.path.realpath(__file__)) + "/salesdata.csv"
        with open(file_path, "rb") as f:
            data = dict(file=(io.BytesIO(f.read()), 'salesdata.csv'))
            response = client.post(
                '/sales_data/upload', data=data,
                headers=headers,
                follow_redirects=True,
                content_type='multipart/form-data'
            )
            assert response.data == b"Sales data uploaded"

        # Query sales data is added
        with app.app_context():
            assert (
                SalesData.query.all()
                is not None
            )

def test_sales_get_total_revenue(auth, client, app):
    with app.app_context():
        populate_users(app)

        # get auth_token
        res = auth.login()
        token = json.loads(res.data.decode())

        headers = {
            "Authorization": "Bearer {}".format(token["access_token"])
        }

        file_path = os.path.dirname(os.path.realpath(__file__)) + "/salesdata.csv"
        with app.app_context():
            with open(file_path) as csv_file:
                csv_reader = csv.reader(csv_file, delimiter=',')
                line_count = 0
                column_mapping_dict = dict()
                for row in csv_reader:
                    if line_count == 0:
                        # Saving column position so the data can be uploaded in any order
                        column_count = 0
                        for column_name in row:
                            column_mapping_dict[column_name] = column_count
                            column_count += 1

                    else:
                        # Add sales data
                        sd = {
                            "customer_name": row[column_mapping_dict["Customer Name"]],
                            "item_description": row[column_mapping_dict["Item Description"]],
                            "item_price": row[column_mapping_dict["Item Price"]],
                            "quantity": row[column_mapping_dict["Quantity"]],
                            "merchant_name": row[column_mapping_dict["Merchant Name"]],
                            "merchant_address": row[column_mapping_dict["Merchant Address"]],
                            "added_by": 1
                        }
                        sales_data = SalesData(**sd)
                        sales_data.save()
                    line_count += 1

            response = client.get('/sales_data/total_revenue', headers=headers)
            assert response.json == {'total_revenue': '526.45'}