import json
import os
import shlex
import sys
import textwrap
import urllib.request

# External modules
import paramiko

# Flintrock modules
from .core import FlintrockCluster
from .ssh import ssh_check_output

FROZEN = getattr(sys, 'frozen', False)

if FROZEN:
    THIS_DIR = sys._MEIPASS
else:
    THIS_DIR = os.path.dirname(os.path.realpath(__file__))

SCRIPTS_DIR = os.path.join(THIS_DIR, 'scripts')

# TODO: Cache these files. (?) They are being read potentially tens or
#       hundreds of times. Maybe it doesn't matter because the files
#       are so small.
def get_formatted_template(path: str, mapping: dict) -> str:
    with open(path) as f:
        formatted = f.read().format(**mapping)
    return formatted

class FlintrockService:
    """
    This is an abstract class. Implementations of this class capture all the logic
    required to fully install and manage services like Spark on Flintrock clusters.
    """

    def __init__(self):
        """
        This is the only method signature that implementations don't have to follow.
        Use this method to set properties like the service version or download source
        which the rest of the methods here will need to do their work.
        """
        raise NotImplementedError

    def install(
            self,
            ssh_client: paramiko.client.SSHClient,
            cluster: FlintrockCluster):
        """
        Install the service on a node via the provided SSH client. This typically
        means downloading a software package and maybe even building it if necessary.

        This method is role-agnostic; it runs on both the cluster master and slaves.
        This method is meant to be called asynchronously.
        """
        raise NotImplementedError

    def configure(
            self,
            ssh_client: paramiko.client.SSHClient,
            cluster: FlintrockCluster):
        """
        Configure the installed service on a node via the provided SSH client. This
        typically means using templates to create configuration files on the node.

        This method is role-agnostic; it runs on both the cluster master and slaves.
        This method is meant to be called asynchronously.
        """
        raise NotImplementedError

    def configure_master(
            self,
            ssh_client: paramiko.client.SSHClient,
            cluster: FlintrockCluster):
        """
        Configure the service master on a node via the provided SSH client after the
        role-agnostic configuration in configure() is complete. Start the master and
        slaves.

        This method is meant to be called once on the cluster master.
        This method is meant to be called asynchronously.
        """
        raise NotImplementedError

    def configure_slave(
            self,
            ssh_client: paramiko.client.SSHClient,
            cluster: FlintrockCluster):
        """
        Configure a service slave on a node via the provided SSH client after the
        role-agnostic configuration in configure() is complete.

        This method is meant to be called once on each cluster slave.
        This method is meant to be called asynchronously.
        """
        raise NotImplementedError

    def health_check(
            self,
            master_host: str):
        """
        Check that the service is up and running by querying the cluster master.
        """
        raise NotImplementedError

class HDFS(FlintrockService):
    def __init__(self, version, download_source):
        self.version = version
        self.download_source = download_source
        self.manifest = {'version': version, 'download_source': download_source}

    def install(
            self,
            ssh_client: paramiko.client.SSHClient,
            cluster: FlintrockCluster):
        print("[{h}] Installing HDFS...".format(
            h=ssh_client.get_transport().getpeername()[0]))

        with ssh_client.open_sftp() as sftp:
            sftp.put(
                localpath=os.path.join(SCRIPTS_DIR, 'download-hadoop.py'),
                remotepath='/tmp/download-hadoop.py')

        ssh_check_output(
            client=ssh_client,
            command="""
                set -e

                python /tmp/download-hadoop.py "{version}" "{download_source}"

                mkdir "hadoop"
                mkdir "hadoop/conf"

                tar xzf "hadoop-{version}.tar.gz" -C "hadoop" --strip-components=1
                rm "hadoop-{version}.tar.gz"

                for f in $(find hadoop/bin -type f -executable -not -name '*.cmd'); do
                    sudo ln -s "$(pwd)/$f" "/usr/local/bin/$(basename $f)"
                done
                echo "export HADOOP_LIBEXEC_DIR='$(pwd)/hadoop/libexec'" >> .bashrc
            """.format(version=self.version, download_source=self.download_source))

    def configure(
            self,
            ssh_client: paramiko.client.SSHClient,
            cluster: FlintrockCluster):
        # TODO: os.walk() through these files.
        template_paths = [
            'hadoop/conf/masters',
            'hadoop/conf/slaves',
            'hadoop/conf/hadoop-env.sh',
            'hadoop/conf/core-site.xml',
            'hadoop/conf/hdfs-site.xml']

        for template_path in template_paths:
            ssh_check_output(
                client=ssh_client,
                command="""
                    echo {f} > {p}
                """.format(
                    f=shlex.quote(
                        get_formatted_template(
                            path=os.path.join(THIS_DIR, "templates", template_path),
                            mapping=cluster.generate_template_mapping(service='hdfs'))),
                    p=shlex.quote(template_path)))

    # TODO: Convert this into start_master() and split master- or slave-specific
    #       stuff out of configure() into configure_master() and configure_slave().
    def configure_master(
            self,
            ssh_client: paramiko.client.SSHClient,
            cluster: FlintrockCluster):
        host = ssh_client.get_transport().getpeername()[0]
        print("[{h}] Configuring HDFS master...".format(h=host))

        ssh_check_output(
            client=ssh_client,
            command="""
                ./hadoop/bin/hdfs namenode -format -nonInteractive
                ./hadoop/sbin/start-dfs.sh
            """)

    def health_check(self, master_host: str):
        # This info is not helpful as a detailed health check, but it gives us
        # an up / not up signal.
        hdfs_master_ui = 'http://{m}:50070/webhdfs/v1/?op=GETCONTENTSUMMARY'.format(m=master_host)

        try:
            hdfs_ui_info = json.loads(
                urllib.request.urlopen(hdfs_master_ui).read().decode('utf-8'))
        except Exception as e:
            # TODO: Catch a more specific problem.
            # TODO: Don't print to screen here. Raise the exception and let the
            #       catcher decide on whether to print.
            print("HDFS health check failed.", file=sys.stderr)
            raise

        print("HDFS online.")

class Spark(FlintrockService):
    def __init__(self, version: str=None, download_source: str=None,
                 git_commit: str=None, git_repository: str=None):
        # TODO: Convert these checks into something that throws a proper exception.
        #       Perhaps reuse logic from CLI.
        assert bool(version) ^ bool(git_commit)
        if git_commit:
            assert git_repository

        self.version = version
        self.download_source = download_source
        self.git_commit = git_commit
        self.git_repository = git_repository

        self.manifest = {
            'version': version,
            'download_source': download_source,
            'git_commit': git_commit,
            'git_repository': git_repository}

    def install(
            self,
            ssh_client: paramiko.client.SSHClient,
            cluster: FlintrockCluster):

        print("[{h}] Installing Spark...".format(
            h=ssh_client.get_transport().getpeername()[0]))

        try:
            if self.version:
                with ssh_client.open_sftp() as sftp:
                    sftp.put(
                        localpath=os.path.join(SCRIPTS_DIR, 'install-spark.sh'),
                        remotepath='/tmp/install-spark.sh')
                    sftp.chmod(path='/tmp/install-spark.sh', mode=0o755)
                url = self.download_source.format(v=self.version)
                ssh_check_output(
                    client=ssh_client,
                    command="""
                        set -e
                        /tmp/install-spark.sh {url}
                        rm -f /tmp/install-spark.sh
                    """.format(url=shlex.quote(url)))
            else:
                ssh_check_output(
                    client=ssh_client,
                    command="""
                        set -e
                        sudo yum install -y git
                        sudo yum install -y java-devel
                        """)
                ssh_check_output(
                    client=ssh_client,
                    command="""
                        set -e
                        git clone {repo} spark
                        cd spark
                        git reset --hard {commit}
                        if [ -e "make-distribution.sh" ]; then
                            ./make-distribution.sh -Phadoop-2.6
                        else
                            ./dev/make-distribution.sh -Phadoop-2.6
                        fi
                    """.format(
                        repo=shlex.quote(self.git_repository),
                        commit=shlex.quote(self.git_commit)))
            ssh_check_output(
                client=ssh_client,
                command="""
                    set -e
                    for f in $(find spark/bin -type f -executable -not -name '*.cmd'); do
                        sudo ln -s "$(pwd)/$f" "/usr/local/bin/$(basename $f)"
                    done
                    echo "export SPARK_HOME='$(pwd)/spark'" >> .bashrc
                """)
        except Exception as e:
            # TODO: This should be a more specific exception.
            print("Error: Failed to install Spark.", file=sys.stderr)
            print(e, file=sys.stderr)
            raise

    def configure(
            self,
            ssh_client: paramiko.client.SSHClient,
            cluster: FlintrockCluster):
        template_paths = [
            'spark/conf/spark-env.sh',
            'spark/conf/slaves']
        for template_path in template_paths:
            ssh_check_output(
                client=ssh_client,
                command="""
                    echo {f} > {p}
                """.format(
                    f=shlex.quote(
                        get_formatted_template(
                            path=os.path.join(THIS_DIR, "templates", template_path),
                            mapping=cluster.generate_template_mapping(service='spark'))),
                    p=shlex.quote(template_path)))

    # TODO: Convert this into start_master() and split master- or slave-specific
    #       stuff out of configure() into configure_master() and configure_slave().
    #       start_slave() can block until slave is fully up; that way we don't need
    #       a sleep() before starting the master.
    def configure_master(
            self,
            ssh_client: paramiko.client.SSHClient,
            cluster: FlintrockCluster):
        host = ssh_client.get_transport().getpeername()[0]
        print("[{h}] Configuring Spark master...".format(h=host))

        # TODO: Maybe move this shell script out to some separate file/folder
        #       for the Spark service.
        # TODO: Add some timeout for waiting on master UI to come up.
        ssh_check_output(
            client=ssh_client,
            command="""
                spark/sbin/start-all.sh

                master_ui_response_code=0
                while [ "$master_ui_response_code" -ne 200 ]; do
                    sleep 1
                    master_ui_response_code="$(
                        curl --head --silent --output /dev/null \
                             --write-out "%{{http_code}}" {m}:8080
                    )"
                done
            """.format(
                m=shlex.quote(cluster.master_host)))

    def health_check(self, master_host: str):
        spark_master_ui = 'http://{m}:8080/json/'.format(m=master_host)

        try:
            spark_ui_info = json.loads(
                urllib.request.urlopen(spark_master_ui).read().decode('utf-8'))
        except Exception as e:
            # TODO: Catch a more specific problem known to be related to Spark not
            #       being up; provide a slightly better error message, and don't
            #       dump a large stack trace on the user.
            print("Spark health check failed.", file=sys.stderr)
            raise

        # TODO: Don't print here. Return this and let the caller print.
        print(textwrap.dedent(
            """\
            Spark Health Report:
              * Master: {status}
              * Workers: {workers}
              * Cores: {cores}
              * Memory: {memory:.1f} GB\
            """.format(
                status=spark_ui_info['status'],
                workers=len(spark_ui_info['workers']),
                cores=spark_ui_info['cores'],
                memory=spark_ui_info['memory'] / 1024)))


from client.models.game_objects.first_player import FirstPlayer
from client.models.game_objects.gorilla import Gorilla
from client.models.game_objects.ladder import Ladder
from client.models.game_objects.platform import Platform
from client.models.game_objects.princess import Princess
from client.models.game_objects.second_player import SecondPlayer
from common.constants import SCENE_HEIGHT, SCENE_GRID_BLOCK_HEIGHT, SCENE_WIDTH, SCENE_GRID_BLOCK_WIDTH
from common.enums.layout_block import LayoutBlock

class GridPainter:
    def __init__(self):
        pass

    """ Paints objects inside layout to a scene """
    def paint_layout(self, scene, layout):
        rows = int(SCENE_HEIGHT / SCENE_GRID_BLOCK_HEIGHT)
        columns = int(SCENE_WIDTH / SCENE_GRID_BLOCK_WIDTH)

        for row in range(rows):
            for column in range(columns):
                if layout[row][column] == LayoutBlock.Platform:
                    temp = Platform()
                    temp.setPos(column * SCENE_GRID_BLOCK_WIDTH, row * SCENE_GRID_BLOCK_HEIGHT)
                    scene.addItem(temp)
                elif layout[row][column] == LayoutBlock.Ladder:
                    temp = Ladder()
                    temp.setPos(column * SCENE_GRID_BLOCK_WIDTH, row * SCENE_GRID_BLOCK_HEIGHT)
                    scene.addItem(temp)
                elif layout[row][column] == LayoutBlock.Player_1:
                    if isinstance(scene.me, FirstPlayer):
                        scene.me.starting_x = column * SCENE_GRID_BLOCK_WIDTH
                        scene.me.starting_y = row * SCENE_GRID_BLOCK_HEIGHT + 5
                        scene.me.item.setPos(scene.me.starting_x, scene.me.starting_y)
                        scene.addItem(scene.me.item)
                    else:
                        scene.opponent.starting_x = column * SCENE_GRID_BLOCK_WIDTH
                        scene.opponent.starting_y = row * SCENE_GRID_BLOCK_HEIGHT + 5
                        scene.opponent.item.setPos(scene.opponent.starting_x, scene.opponent.starting_y)
                        scene.addItem(scene.opponent.item)
                    layout[row][column] = None
                elif layout[row][column] == LayoutBlock.Player_2:
                    if isinstance(scene.me, SecondPlayer):
                        scene.me.starting_x = column * SCENE_GRID_BLOCK_WIDTH + 13
                        scene.me.starting_y = row * SCENE_GRID_BLOCK_HEIGHT + 5
                        scene.me.item.setPos(scene.me.starting_x, scene.me.starting_y)
                        scene.addItem(scene.me.item)
                    else:
                        scene.opponent.starting_x = column * SCENE_GRID_BLOCK_WIDTH + 13
                        scene.opponent.starting_y = row * SCENE_GRID_BLOCK_HEIGHT + 5
                        scene.opponent.item.setPos(scene.opponent.starting_x, scene.opponent.starting_y)
                        scene.addItem(scene.opponent.item)
                    layout[row][column] = None
                elif layout[row][column] == LayoutBlock.Princess:
                    scene.princess = Princess(scene, column * SCENE_GRID_BLOCK_WIDTH, row * SCENE_GRID_BLOCK_HEIGHT)
                    scene.addItem(scene.princess.item)
                elif layout[row][column] == LayoutBlock.Gorilla:
                    scene.gorilla = Gorilla(column * SCENE_GRID_BLOCK_WIDTH, row * SCENE_GRID_BLOCK_HEIGHT - 15)
                    scene.addItem(scene.gorilla.item)


import logging

from aiogram import Router
from aiogram.filters import Command
from aiogram.types import Message

from src.application.postman.dto import DispatchMessage
from src.application.postman.service import PostmanService
from src.application.subscribers.dto import Subscriber
from src.application.subscribers.interfaces import SubscriberRepo

logger = logging.getLogger(__name__)
router = Router()

@router.message(Command("subscribe"))
async def command_subscribe(message: Message, sub_repo: SubscriberRepo) -> None:
    sub: Subscriber | None = await sub_repo.get_by_username(message.from_user.username)

    if sub is not None:
        await message.reply("Already subscribed!")
        return

    sub = Subscriber(
        telegram_id=str(message.from_user.id), telegram_username=message.from_user.username
    )
    await sub_repo.create(sub)
    await message.reply("Subscribed!")

@router.message(Command("send"))
async def command_start_dispatch(message: Message, postman: PostmanService) -> None:
    msg = DispatchMessage(
        from_username=message.from_user.username, message=message.text
    )
    logger.info(f"Performing dispatch: {msg!r}")
    await postman.perform_dispatch(msg)


from OANDA_Historical import *
import statsmodels.formula.api as smf
#print(api.get_instruments())

SPX = OANDA_Base(symbol='SPX500_USD', start= monthsAgo(120), end= lastDay(), granularity='D', price='M', amount=10000, ftc=0.000, ptc=0.0008, verbos=True)
DAX = OANDA_Base(symbol='DE30_EUR', start= monthsAgo(120), end= lastDay(), granularity='D', price='M', amount=10000, ftc=0.000, ptc=0.0008, verbos=True)
df = pd.DataFrame()
df['SPX'] = SPX.get_data()['return']
df['DAX'] = DAX.get_data()['return']

''' plot

df.plot(subplots=True, grid=True, style='b',figsize=(8,6))
plt.show()
'''

result = smf.ols(formula="SPX ~ DAX", data=df).fit()
print(result.params)
print(result.summary())

plt.plot(df['SPX'], df['DAX'], 'r.')
ax = plt.axis()
x = np.linspace(ax[0], ax[1] +0.01)
plt.plot(x, result.params[0] + result.params[1] * x, 'b', lw=2)
plt.grid(True)
plt.axis('tight')
plt.show()

print(df.corr())

df.rolling(50).corr(pairwise=True).plot(grid=True,style='b')
plt.show()


import json
import datetime
import time
import os
import dateutil.parser
import logging
import re
from app import get_data
import boto3

#pointing chatbot database now
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)

def clean_phone_no(a):
    a = re.sub('[-()!@+#$]', '', a)
    a = a.replace(' ', '')

    if len(a) < 10:
        return 0

    if len(a) == 10:
        return '+1' + a

    if len(a) == 11:
        if a[:1] == '+':
            return '+1' + a[:10]

        else:
            return '+' + a[:11]

    return a

def setup(the_email):
    client = boto3.client('sns')

    data = get_data('sp_confirm', [the_email])
    logger.debug(data)

    if (data):
        a = data[0]

        starttime = a[4].strftime('%I:%M %p')
        endtime = a[5].strftime('%I:%M %p')
        theday = a[4].strftime('%Y-%m-%d')
        cleanphone = clean_phone_no(a[2])
        therole = a[9]

        msg = 'Hi '+ a[0] +'!  Thank you for scheduling your interview.  We have you booked on ' + theday + ' from ' + starttime + ' to ' + endtime + ' with ' + a[6] + ' in ' + a[7] + ' for the ' + therole +' role.  See you then!'

        if (a[3]=='both' or a[3]=='sms'):
            client.publish(
                PhoneNumber= cleanphone,
                Message= msg
            )

        if (a[3]=='both' or a[3]=='email'):
            send_one_email(a[1], msg, 'Interview Confirmation')
    else:
        logger.debug('nothing for this email ...' + str(the_email))

def send_one_email(theemail, msg, thetitle):
    client = boto3.client('ses')

    response = client.send_email(
        Destination={
            'BccAddresses': [
            ],
            'ToAddresses': [
                theemail,
            ],
        },
        Message={
            'Body': {
                'Html': {
                    'Charset': 'UTF-8',
                    'Data': msg,
                },
                'Text': {
                    'Charset': 'UTF-8',
                    'Data': msg,
                },
            },
            'Subject': {
                'Charset': 'UTF-8',
                'Data': thetitle,
            },
        },
        Source='info@adri-sys.com',
    )

def lambda_handler(event, context):

    logger.debug('this is the event')
    logger.debug(event)
    logger.debug('this is the context ...')
    logger.debug(context)

    the_email = event['emailaddress']

    setup(the_email)

    return event


import torch
import torch.nn as nn
import torch.nn.functional as F
from torchcrf import CRF
from w2v import load_word2vec

class BiLSTM_CRF(nn.Module):
    def __init__(self,
                 word2seq,
                 char2seq,
                 tag2seq,
                 config,
                 num_heads=8):
        super(BiLSTM_CRF, self).__init__()
        self.word2seq = word2seq
        self.char2seq = char2seq
        self.tag2seq = tag2seq
        self.config = config

        self.char_embedding = nn.Embedding(len(self.char2seq), self.config.char_embedding_dim)
        self.char_cnn = nn.Conv2d(in_channels=self.config.max_char_len, out_channels=1, kernel_size=3, stride=1, padding=1)

        if self.config.use_pretrain_embedding:
            self.embedding = nn.Embedding.from_pretrained(load_word2vec(self.config.embedding_path, word2seq), freeze=False)
        else:
            self.embedding = nn.Embedding(len(self.word2seq), self.config.embedding_dim)
        self.lstm = nn.LSTM(input_size=self.config.embedding_dim + self.config.char_embedding_dim, hidden_size=self.config.hidden_size, batch_first=True, bidirectional=True)
        self.dropout = nn.Dropout(p=self.config.dropout)
        self.fc = nn.Linear(self.config.hidden_size * 2, len(self.tag2seq))
        if self.config.use_crf:
            self.crf = CRF(num_tags=len(self.tag2seq), batch_first=True)
        if self.config.use_att:
            self.mul_att = nn.MultiheadAttention(embed_dim=self.config.hidden_size * 2, batch_first=True, dropout=self.config.dropout, num_heads=num_heads)

    def encode(self, sentences):
        output, (h_n, _) = self.lstm(sentences)  # output [batch_size, length, hidden_size * 2] h_n [num_lays*num_direction, batch_size, hidden_size]
        output = self.dropout(output)
        if self.config.use_att:
            output, _ = self.mul_att(output, output, output)
        emit_score = self.fc(output)  # emit_score[batch_size, length, len(tag2seq)]
        return emit_score

    def pad_result(self, pred, seq_len):
        for pred_tag in pred:
            if len(pred_tag) < seq_len:
                pred_tag += [0] * (seq_len - len(pred_tag))
        pred = torch.tensor(pred).to(self.config.device)
        return pred

    def forward(self, sentences, chars, tags=None):
        """
        计算loss
        sentences:[batch_size, max_len]
        chars: [batch_size, max_len, max_char_len]
        tags:[batch_size, max_len]
        """
        seq_len = sentences.size()[-1]
        mask = sentences != 1
        sentences = self.embedding(sentences)  # sentences[batch_size, max_len, embedding_dim]
        chars = self.char_embedding(chars).transpose(1, 2)  # chars: [batch_size, max_len, max_char_len, char_embedding_dim]
        chars = self.char_cnn(chars).squeeze(1)
        sentences = torch.cat((sentences, chars), -1)
        emit_score = self.encode(sentences)
        if tags is None:
            if self.config.use_crf:
                pred = self.crf.decode(emissions=emit_score, mask=mask)
                pred = self.pad_result(pred, seq_len)
            else:
                pred = emit_score.transpose(1, 2)
                pred = pred.argmax(1)
            return pred
        else:
            if self.config.use_crf:
                loss = self.crf(emissions=emit_score, tags=tags, mask=mask) * -1
            else:
                loss = torch.nn.functional.cross_entropy(emit_score.transpose(1, 2), tags)
            return loss


from collections import defaultdict
def solution(fees, records):
    answer = []
    dic = defaultdict(list)
    fee = [0] * 9999

    for record in records:
        time, num, status = record.split()
        if status == "IN":
            dic[num].append(time)
        else:
            xh, xm = dic[num][-1].split(":")
            yh, ym = time.split(":")
            dic[num].pop()
            zh = int(yh) - int(xh)
            zm = int(ym) - int(xm)
            temp = (zh * 60) + zm
            fee[int(num)] += temp

    for d in dic:
        if dic[d]:
            xh, xm = dic[d][-1].split(":")
            temp = ((23-int(xh)) * 60) + (59-int(xm))
            fee[int(d)] += temp

    for f in fee:
        if f != 0:
            if f <= fees[0]:
                answer.append(fees[1])
            else:
                if (f-fees[0]) % fees[2] == 0:
                    answer.append(fees[1] + (((f-fees[0])//fees[2]) * fees[3]))
                else:
                    answer.append(fees[1] + (((f-fees[0])//fees[2]+ 1) * fees[3]))
    return answer

print(solution([180, 5000, 10, 600],["05:34 5961 IN", "06:00 0000 IN", "06:34 0000 OUT", "07:59 5961 OUT", "07:59 0148 IN", "18:59 0000 IN", "19:09 0148 OUT", "22:59 5961 IN", "23:00 5961 OUT"]))


# -*- coding: utf-8 -*-
"""
Created on Fri Dec 30 15:59:31 2016

@author: luzhangqin
"""

import matplotlib.pylab as plt
from matplotlib.font_manager import FontProperties

from sklearn.linear_model import LinearRegression

#显示中文
font = FontProperties(fname = r'/usr/share/fonts/truetype/arphic/ukai.ttc', size = 10)

def runplt():
    #创建一个用来显示图形输出的一个窗口对象
    plt.figure()
    plt.title('批萨价格与直径数据', fontproperties = font)
    plt.xlabel('直径（英寸）', fontproperties = font)
    plt.ylabel('价格（美元）', fontproperties = font)
    #[xmin, xmax, ymin, ymax]
    plt.axis([0, 25, 0, 25])
    #网格
    plt.grid(True)
    return plt

if __name__ == '__main__':
    plt = runplt()
    x = [[6], [8], [10], [14], [18]]
    y = [[7], [9], [13], [17.5], [18]]
    #测试集
    x_test = [[8], [9], [11], [16], [12]]
    y_test = [[11], [8.5], [15], [18], [11]]

    model = LinearRegression()
    model.fit(x,y)
    #R方
    r2 = model.score(x_test, y_test)
    print(r2)


import messagebird
import sqlite3
import os

def fetch_contacts(conn):
    cursor = conn.cursor()
    cursor.execute(
        "SELECT c.phone FROM contacts as c WHERE c.phone LIKE '57%' AND  c.city LIKE 'Bogota' AND c.sent IS FALSE LIMIT 10")
    # results = [phone.split('57')[-1] for (phone,) in cursor.fetchall()]
    results = [phone for (phone,) in cursor.fetchall()]
    cursor.close()
    return results

def send_sms(conn, client, phone):
    try:
        client.message_create(
            'MessageBird',
            '+' + phone,
            os.getenv('MESSAGE_BODY')
        )

        cursor = conn.cursor()
        cursor.execute(
            "UPDATE contacts SET sent = TRUE WHERE phone like ?", (phone,))
        conn.commit()
        print('sms sent')
    except messagebird.client.ErrorException as e:
        print(e)
        print('Error sending the sms')

if __name__ == '__main__':
    conn = sqlite3.connect('contacts.db', timeout=10)
    contacts = fetch_contacts(conn)
    key = os.getenv('MESSAGEBIRD_ACCESS_KEY')
    client = messagebird.Client(key)
    for contact in contacts:
        print(contact)
        send_sms(conn, client, contact)


import os

from .file import *
from .transform import one_hot_encode_data, merge_splits, split_data, scale_data
from .uci import download_UCI
from .torch import get_torchvision_data

def load_data(source="local", split_vars = None, merge_before_split=False, one_hot_encode=True, **kwargs):
	dataloader = select_dataloader(source)

	#using kwargs["data"] instead of data, cause data may be inside kwargs in case from=="as_param" may avoid duplication?
	data = dataloader(**kwargs)

	if split_vars is not None: data = split_vars(data)

	if merge_before_split: merge_splits(**kwargs)

	data = split_data(data, **kwargs)

	data, data_scaler = scale_data(data, **kwargs)

	if one_hot_encode: data = one_hot_encode_data(data, **kwargs)

	if data_scaler is None: return data
	return data, data_scaler

def select_dataloader(source):
	###Select data loading
	loc = source.lower()
	if loc == "uci":
		dataloader = download_UCI
	# elif kwargs["from"].lower() == "tfds":
	# 	dataloader = get_tfds_data
	elif loc == "torchvision":
		dataloader = get_torchvision_data
	elif loc == "local":
		dataloader = get_local_data
	elif loc == "as_param":
		dataloader = lambda *args, **kwargs: kwargs["data"]
	elif callable(loc):
		dataloader = loc
	else:
		raise NotImplementedError("DATA IMPORT NOT IMPLEMENTED FROM", loc)
	return dataloader

def get_local_data(name, data_folder = "../data/", loader_params = {}, **kwargs):
	filename = os.path.join(data_folder,name)
	ext = filename.split(".")[-1]
	if ext=="npz": #already a dict
		dct = load_arrays(filename, loader_params, **kwargs)
		return dct
	# elif ext in ["pkl","pickle"]: #already a dict
	# 	dct = load_pickle(filename)
	# 	return dct
	else:
		if ext=="csv":
			app = load_csv(filename, loader_params, **kwargs)
		elif ext=="npy":
			app = load_numpy(filename, loader_params, **kwargs)
		elif ext=="npz":
			app = load_npz(filename, loader_params, **kwargs)
		return {"x": app}
	raise NotImplementedError


"""Advent of Code 2019, day 21: https://adventofcode.com/2019/day/21"""
import logging

from advent_of_code.aoc2019.intcode import IntcodeMachine
from advent_of_code.base import Solution

log = logging.getLogger("aoc")

class AocSolution(Solution[int, int]):
    def __init__(self, **kwargs):
        super().__init__(21, 2019, **kwargs)

    def run_springscript(self, script: list[str]) -> int:
        with self.open_input() as f:
            program = IntcodeMachine.read_buffer(f)
        script_bytes = ("\n".join(script) + "\n").encode("ASCII")
        machine = IntcodeMachine(program, iter(script_bytes))
        result = list(machine.run())
        if result[-1] == 10:
            log.debug(bytes(result).decode("ASCII"))
            raise SpringScriptError()
        return result[-1]

    def solve_part_one(self) -> int:
        script = [
            "OR D J",  # Jump if we can.
            "OR A T",  # Unless...
            "AND B T",
            "AND C T",
            "NOT T T",
            "AND T J",  # A, B, and C are all True.
            "WALK",
        ]
        return self.run_springscript(script)

    def solve_part_two(self) -> int:
        script = [
            "OR D J",  # Jump if we can.
            "OR A T",  # Unless...
            "AND B T",
            "AND C T",
            "NOT T T",
            "AND T J",  # A, B, and C are all True.
            "NOT A T",  # (Reset T -> False)
            "AND A T",
            "OR E T",  # Or...
            "OR H T",
            "AND T J",  # E and H are both False.
            "RUN",
        ]
        return self.run_springscript(script)

class SpringScriptError(Exception):
    ...


from flask_restful import Resource, abort

from app.db import db, Contract, Proposal, Challenge
from app.comm.decorators import with_poll_contract
from app.util.reqargs import with_args, Argument
from app.util.types import address
from app.util.abi_data import create_params_data
from app.util.eth_call import (
    get_community_token_address,
    get_community_token_address,
    eth_call,
)
from eth_utils import to_hex, to_bytes
from eth_abi import decode_single

from decimal import Decimal

class VotingPower(Resource):
    path = "/voting/<addr>/<int:on_chain_id>/voting-power/<voter>"

    @with_poll_contract(addr="addr")
    def get(self, addr, on_chain_id, voter, poll_contract):
        voting_snapshot = 0
        contract = (
            db.session.query(Contract)
            .filter_by(address=poll_contract.address)
            .one_or_none()
        )

        if contract.contract_type == "TCR":
            challenge = db.session.query(Challenge).filter_by(
                tcr_address=poll_contract.address, challenge_id=on_chain_id
            ).one_or_none() or abort(400, message=f"Challenge not found")
            voting_snapshot = challenge.token_snap_shot
            token_address = get_community_token_address(poll_contract.address)
        elif contract.contract_type == "PARAMETER":
            proposal = db.session.query(Proposal).filter_by(
                parameter_address=poll_contract.address, proposal_id=on_chain_id
            ).one_or_none() or abort(400, message=f"Challenge not found")
            voting_snapshot = proposal.token_snap_shot
            token_address = get_community_token_address(addr)
        else:
            abort(400)

        voting_power = Decimal(
            decode_single(
                "uint256",
                to_bytes(
                    hexstr=eth_call(
                        token_address,
                        "historicalVotingPowerAtNonce(address,uint256)",
                        to_hex(
                            create_params_data(
                                ("address", voter), ("uint256", voting_snapshot)
                            )
                        ),
                    )
                ),
            )
        )
        return {
            "result": {
                "votingPower": voting_power,
                "votingSnapShot": voting_snapshot,
            }
        }


from scrapy.dupefilters import BaseDupeFilter, referer_str
import logging
import hashlib
from scrapy.utils.python import to_bytes

class MyRFPDupeFilter(BaseDupeFilter):
    """My Request Fingerprint duplicates filter"""

    def __init__(self, debug=False):
        self.fingerprints = set()
        self.logdupes = True
        self.debug = debug
        self.logger = logging.getLogger(__name__)

    @classmethod
    def from_settings(cls, settings):
        debug = settings.getbool('DUPEFILTER_DEBUG')
        return cls(debug)

    def request_seen(self, request):
        fp = self.request_fingerprint(request)
        if fp in self.fingerprints:
            return True
        self.fingerprints.add(fp)

    def request_fingerprint(self, request) -> str:
        return hashlib.md5(to_bytes(request.url)).hexdigest()

    def close(self, reason):
        msg = "The number of URLs: %(url_num)d"
        args = {'url_num': len(self.fingerprints)}
        self.logger.info(msg, args)

    def log(self, request, spider):
        if self.debug:
            msg = "Filtered duplicate request: %(request)s (referer: %(referer)s)"
            args = {'request': request, 'referer': referer_str(request)}
            self.logger.debug(msg, args, extra={'spider': spider})
        elif self.logdupes:
            msg = ("Filtered duplicate request: %(request)s"
                   " - no more duplicates will be shown"
                   " (see DUPEFILTER_DEBUG to show all duplicates)")
            self.logger.debug(msg, {'request': request}, extra={'spider': spider})
            self.logdupes = False

        spider.crawler.stats.inc_value('dupefilter/filtered', spider=spider)


import multiprocessing
import time

def depositMoney(money,lock):
    for i in range(50):
        lock.acquire()
        money.value+=1
        lock.release()
        print("Deposit!")

def withdrawMoney(money,lock):
    for i in range(50):
        lock.acquire()
        money.value-=1
        lock.release()
        print("Withdrawn!")

if __name__ == "__main__":
    balanceMoney = multiprocessing.Value('i',0)
    lockProcess=multiprocessing.Lock()
    process1=multiprocessing.Process(target=depositMoney, args=(balanceMoney,lockProcess))
    process2=multiprocessing.Process(target=withdrawMoney, args=(balanceMoney,lockProcess))

    process1.start()
    process2.start()
    process1.join()
    process2.join()

    print("Final Balance: ",balanceMoney.value)


import json
import time
import datetime

def timestamp():
    current_time = datetime.datetime.now()
    formatted_time = current_time.strftime('%Y-%m-%d %H:%M:%S')
    return formatted_time

def write_file(pairs):
    try:
        print(f"{timestamp()} Write pairs to file {len(pairs)} pairs")
        filename = f"./pairs/pairs-{int(time.time() * 1000)}.json"
        with open(filename, 'w') as f:
            json.dump(pairs, f)
        return
    except Exception as e:
        print(e)
        raise e


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import sys

from tensorflow.examples.tutorials.mnist import input_data

import tensorflow as tf

FLAGS = None

# Weight and bias

def weight_variable(shape):
    """
    Initialize weight matrices with truncated normal distribution with standard deviation of 0.1.

    In general, shape = [filter_height, filter_width, in_channels, out_channels].
    """
    init = tf.truncated_normal(shape, stddev=0.1)
    return tf.Variable(init)

def bias_variable(shape):
    """
    Initialize bias vectors with constant value of 0.1.

    In general, shape = [out_channels].
    """
    init = tf.constant(0.1, shape=shape)
    return tf.Variable(init)

# Layers

def conv2d(x, W):
    """2D convolution of x with filter W, unit stride."""
    return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')

def maxpool2d(x):
    """2x2 max pooling layer with non-overlapping kernel stride."""
    return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')

def flatten(x):
    size = 1
    for dim in x.shape.as_list()[1:]:
        size *= dim
    return tf.reshape(x, [-1, size])

def fullconn(x, W):
    return tf.matmul(x, W)

# Model

def model(x_image, x_image_shape = [28, 28]):
    # Layer 0 (2D convolution layer)
    filter_shape_0 = [10, 10]
    in_channels_0 = 1
    out_channels_0 = 32

    # Reshape input
    batch = -1
    x_0 = tf.reshape(x_image, [batch] + list(x_image_shape) + [in_channels_0])

    w_0 = weight_variable(filter_shape_0 + [in_channels_0, out_channels_0])
    b_0 = bias_variable([out_channels_0])

    z_0 = conv2d(x_0, w_0) + b_0

    x_1 = tf.nn.relu(z_0)

    # Layer 1 (Max pool 2x2)
    in_channels_1 = out_channels_0
    out_channels_1 = in_channels_1

    x_2 = maxpool2d(x_1)

    # Layer 2 (2D convolution layer)
    filter_shape_2 = [5, 5]
    in_channels_2 = out_channels_1
    out_channels_2 = 16

    w_2 = weight_variable(filter_shape_2 + [in_channels_2, out_channels_2])
    b_2 = bias_variable([out_channels_2])

    z_2 = conv2d(x_2, w_2) + b_2

    x_3 = tf.nn.relu(z_2)

    # Layer 3 (Max pool 2x2)
    in_channels_3 = out_channels_2
    out_channels_3 = in_channels_3

    x_4 = maxpool2d(x_3)

    # Layer 4 (Flatten)
    in_channels_4 = out_channels_3
    out_channels_4 = in_channels_4

    x_5 = flatten(x_4)

    # Layer 5 (Fully connected layer)
    in_channels_5 = x_5.shape.as_list()[-1]
    out_channels_5 = 1024

    w_5 = weight_variable([in_channels_5, out_channels_5])
    b_5 = bias_variable([out_channels_5])

    z_5 = fullconn(x_5, w_5) + b_5

    x_6 = tf.nn.relu(z_5)

    # Layer 6 (Fully connected layer)
    in_channels_6 = out_channels_5
    out_channels_6 = 10

    w_6 = weight_variable([in_channels_6, out_channels_6])
    b_6 = bias_variable([out_channels_6])

    z_6 = fullconn(x_6, w_6) + b_6

    return z_6

def main(_):
    # Open output files
    train_stream = open("train_accuracy.csv", "w")
    print("Epoch,Train Accuracy\n", file=train_stream)

    test_stream = open("test_accuracy.csv", "w")
    print("Epoch,Test Accuracy\n", file=test_stream)

    # Import data
    mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True)

    # Create the model
    x = tf.placeholder(tf.float32, [None, 784])
    y = model(x)

    # Define loss and optimizer
    y_ = tf.placeholder(tf.float32, [None, 10])

    # The raw formulation of cross-entropy,
    #
    #   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),
    #                                 reduction_indices=[1]))
    #
    # can be numerically unstable.
    #
    # So here we use tf.nn.softmax_cross_entropy_with_logits on the raw
    # outputs of 'y', and then average across the batch.

    cross_entropy = tf.reduce_mean(
            tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))
    train_step = tf.train.AdagradOptimizer(1e-3).minimize(cross_entropy)
    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

    sess = tf.InteractiveSession()
    sess.run(tf.global_variables_initializer())

    # Train
    batch_size = 100
    epochs = 200
    batch_num = 1
    while mnist.train.epochs_completed < epochs:
        prev_nepoch = mnist.train.epochs_completed
        batch_xs, batch_ys = mnist.train.next_batch(batch_size)
        if mnist.train.epochs_completed > prev_nepoch:
            train_accuracy = sess.run(accuracy, feed_dict={x: mnist.train.images, y_: mnist.train.labels})
            print("{},{}\n".format(prev_nepoch, train_accuracy), file=train_stream)
            test_accuracy = sess.run(accuracy, feed_dict={x: mnist.test.images, y_: mnist.test.labels})
            print("{},{}\n".format(prev_nepoch, test_accuracy), file=test_stream)
            batch_num = 1
        sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})
        if batch_num % 100 == 0:
            batch_accuracy = sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys})
            print("Epoch: {0}, Batch: {1}, Accuracy: {2:.4f}".format(mnist.train.epochs_completed + 1, batch_num, batch_accuracy))
        batch_num += 1

    # Test trained model
    test_accuracy = sess.run(accuracy, feed_dict={x: mnist.test.images, y_: mnist.test.labels})
    print("\n Test Accuracy: {0:.4f}".format(test_accuracy))

    train_stream.close()
    test_stream.close()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--data_dir', type=str, default='/tmp/tensorflow/mnist/input_data',
                                            help='Directory for storing input data')
    FLAGS, unparsed = parser.parse_known_args()
    tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)


from matplotlib import pyplot
import numpy as np
import os
import shutil
import caffe2.python.predictor.predictor_exporter as pe

from caffe2.python import (
    brew,
    core,
    model_helper,
    net_drawer,
    optimizer,
    visualize,
    workspace,
)

# If you would like to see some really detailed initializations,
# you can change --caffe2_log_level=0 to --caffe2_log_level=-1
core.GlobalInit(['caffe2', '--caffe2_log_level=0'])
print("Necessities imported!")

# If True, a more complicated convolutional model is used
# If False, a multilayer perceptron model is used
USE_LENET_MODEL = True

# This section preps your image and test set in a lmdb database
def DownloadResource(url, path):
    '''Downloads resources from s3 by url and unzips them to the provided path'''
    import requests, zipfile, io
    print(("Downloading... {} to {}".format(url, path)))
    r = requests.get(url, stream=True)
    z = zipfile.ZipFile(io.StringIO(r.content))
    z.extractall(path)
    print("Completed download and extraction.")

current_folder = os.path.join(os.path.expanduser('~'), 'caffe2_notebooks')
data_folder = os.path.join(current_folder, 'tutorial_data', 'mnist')
root_folder = os.path.join(current_folder, 'tutorial_files', 'tutorial_mnist')
db_missing = False

if not os.path.exists(data_folder):
    os.makedirs(data_folder)
    print(("Your data folder was not found!! This was generated: {}".format(data_folder)))

# Look for existing database: lmdb
if os.path.exists(os.path.join(data_folder,"mnist-train-nchw-lmdb")):
    print("lmdb train db found!")
else:
    db_missing = True

if os.path.exists(os.path.join(data_folder,"mnist-test-nchw-lmdb")):
    print("lmdb test db found!")
else:
    db_missing = True

# attempt the download of the db if either was missing
if db_missing:
    print("one or both of the MNIST lmbd dbs not found!!")
    db_url = "http://download.caffe2.ai/databases/mnist-lmdb.zip"
    try:
        DownloadResource(db_url, data_folder)
    except Exception as ex:
        print(("Failed to download dataset. Please download it manually from {}".format(db_url)))
        print(("Unzip it and place the two database folders here: {}".format(data_folder)))
        raise ex

if os.path.exists(root_folder):
    print("Looks like you ran this before, so we need to cleanup those old files...")
    shutil.rmtree(root_folder)

os.makedirs(root_folder)
workspace.ResetWorkspace(root_folder)

print(("training data folder:" + data_folder))
print(("workspace root folder:" + root_folder))

def AddInput(model, batch_size, db, db_type):
    # load the data
    data_uint8, label = brew.db_input(
        model,
        blobs_out=["data_uint8", "label"],
        batch_size=batch_size,
        db=db,
        db_type=db_type,
    )
    # cast the data to float
    data = model.Cast(data_uint8, "data", to=core.DataType.FLOAT)
    # scale data from [0,255] down to [0,1]
    data = model.Scale(data, data, scale=float(1./256))
    # don't need the gradient for the backward pass
    data = model.StopGradient(data, data)
    return data, label

def AddMLPModel(model, data):
    size = 28 * 28 * 1
    sizes = [size, size * 2, size * 2, 10]
    layer = data
    for i in range(len(sizes) - 1):
        layer = brew.fc(model, layer, 'dense_{}'.format(i), dim_in=sizes[i], dim_out=sizes[i + 1])
        layer = model.net.Relu(layer, 'relu_{}'.format(i))
    softmax = model.net.Softmax(layer, 'softmax')
    return softmax

def AddLeNetModel(model, data):
    '''
    This part is the standard LeNet model: from data to the softmax prediction.

    For each convolutional layer we specify dim_in - number of input channels
    and dim_out - number or output channels. Also each Conv and MaxPool layer changes the
    image size. For example, kernel of size 5 reduces each side of an image by 4.

    While when we have kernel and stride sizes equal 2 in a MaxPool layer, it divides
    each side in half.
    '''
    # Image size: 28 x 28 -> 24 x 24
    conv1 = brew.conv(model, data, 'conv1', dim_in=1, dim_out=20, kernel=5)
    # Image size: 24 x 24 -> 12 x 12
    pool1 = model.net.MaxPool(conv1, 'pool1', kernel=2, stride=2)
    # Image size: 12 x 12 -> 8 x 8
    conv2 = brew.conv(model, pool1, 'conv2', dim_in=20, dim_out=50, kernel=5)
    # Image size: 8 x 8 -> 4 x 4
    pool2 = model.net.MaxPool(conv2, 'pool2', kernel=2, stride=2)
    # 50 * 4 * 4 stands for dim_out from previous layer multiplied by the image size
    fc3 = brew.fc(model, pool2, 'fc3', dim_in=50 * 4 * 4, dim_out=500)
    fc3 = model.net.Relu(fc3, 'relu3')
    pred = brew.fc(model, fc3, 'pred', 500, 10)
    softmax = model.net.Softmax(pred, 'softmax')
    return softmax

def AddModel(model, data):
    if USE_LENET_MODEL:
        return AddLeNetModel(model, data)
    else:
        return AddMLPModel(model, data)

def AddAccuracy(model, softmax, label):
    """Adds an accuracy op to the model"""
    accuracy = model.Accuracy([softmax, label], "accuracy")
    return accuracy

def AddTrainingOperators(model, softmax, label):
    """Adds training operators to the model."""
    xent = model.LabelCrossEntropy([softmax, label], 'xent')
    # compute the expected loss
    loss = model.AveragedLoss(xent, "loss")
    # track the accuracy of the model
    AddAccuracy(model, softmax, label)
    # use the average loss we just computed to add gradient operators to the model
    model.AddGradientOperators([loss])
    optimizer.build_sgd(
        model,
        base_learning_rate=0.1,
        policy="step",
        stepsize=1,
        gamma=0.999,
    )

def AddBookkeepingOperators(model):
    """This adds a few bookkeeping operators that we can inspect later.

    These operators do not affect the training procedure: they only collect
    statistics and prints them to file or to logs.
    """
    # Print basically prints out the content of the blob. to_file=1 routes the
    # printed output to a file. The file is going to be stored under
    #     root_folder/[blob name]
    model.Print('accuracy', [], to_file=1)
    model.Print('loss', [], to_file=1)
    # Summarizes the parameters. Different from Print, Summarize gives some
    # statistics of the parameter, such as mean, std, min and max.
    for param in model.params:
        model.Summarize(param, [], to_file=1)
        model.Summarize(model.param_to_grad[param], [], to_file=1)
    # Now, if we really want to be verbose, we can summarize EVERY blob
    # that the model produces; it is probably not a good idea, because that
    # is going to take time - summarization do not come for free. For this
    # demo, we will only show how to summarize the parameters and their
    # gradients.

arg_scope = {"order": "NCHW"}
train_model = model_helper.ModelHelper(name="mnist_train", arg_scope=arg_scope)
data, label = AddInput(
    train_model, batch_size=64,
    db=os.path.join(data_folder, 'mnist-train-nchw-lmdb'),
    db_type='lmdb')
softmax = AddModel(train_model, data)
AddTrainingOperators(train_model, softmax, label)
AddBookkeepingOperators(train_model)

# Testing model. We will set the batch size to 100, so that the testing
# pass is 100 iterations (10,000 images in total).
# For the testing model, we need the data input part, the main AddModel
# part, and an accuracy part. Note that init_params is set False because
# we will be using the parameters obtained from the train model.
test_model = model_helper.ModelHelper(
    name="mnist_test", arg_scope=arg_scope, init_params=False)
data, label = AddInput(
    test_model, batch_size=100,
    db=os.path.join(data_folder, 'mnist-test-nchw-lmdb'),
    db_type='lmdb')
softmax = AddModel(test_model, data)
AddAccuracy(test_model, softmax, label)

# Deployment model. We simply need the main AddModel part.
deploy_model = model_helper.ModelHelper(
    name="mnist_deploy", arg_scope=arg_scope, init_params=False)
AddModel(deploy_model, "data")
# You may wonder what happens with the param_init_net part of the deploy_model.
# No, we will not use them, since during deployment time we will not randomly
# initialize the parameters, but load the parameters from the db.

print(str(train_model.net.Proto())[:400] + '\n...')
print(str(train_model.param_init_net.Proto())[:400] + '\n...')

# The parameter initialization network only needs to be run once.
# Now all the parameter blobs are going to be initialized in the workspace.
workspace.RunNetOnce(train_model.param_init_net)

# Creating an actual network as a C++ object in memory.
# We need this as its going to be used a lot.
# So we avoid an object every single time it is used.

# overwrite=True allows you to run this cell several times and avoid errors
workspace.CreateNet(train_model.net, overwrite=True)

# Set the iterations number and track the accuracy & loss
total_iters = 200
accuracy = np.zeros(total_iters)
loss = np.zeros(total_iters)

# Now, we will manually run the network for 200 iterations.
for i in range(total_iters):
    workspace.RunNet(train_model.net)
    accuracy[i] = workspace.blobs['accuracy']
    loss[i] = workspace.blobs['loss']

# After the execution is done, let's plot the values.
pyplot.plot(loss, 'b')
pyplot.plot(accuracy, 'r')
pyplot.legend(('Loss', 'Accuracy'), loc='upper right')
pyplot.show()

# Let's look at some of the data.
pyplot.figure()
data = workspace.FetchBlob('data')
_ = visualize.NCHW.ShowMultiple(data)
pyplot.figure()
softmax = workspace.FetchBlob('softmax')
_ = pyplot.plot(softmax[0], 'ro')
pyplot.title('Prediction for the first image')
pyplot.show()

if USE_LENET_MODEL:
    pyplot.figure()
    # We look into the first conv layer output. Change this to conv2 in order to look into the second one.
    conv = workspace.FetchBlob('conv1')

    # We can look into any channel. Think of it as a feature model learned.
    # In this case we look into the 5th channel. Play with other channels to see other features
    conv = conv[:,[5],:,:]

    _ = visualize.NCHW.ShowMultiple(conv)

    pyplot.show()

# param_init_net here will only create a data reader
# Other parameters won't be re-created because we selected init_params=False before
workspace.RunNetOnce(test_model.param_init_net)
workspace.CreateNet(test_model.net, overwrite=True)
test_accuracy = np.zeros(100)
for i in range(100):
    workspace.RunNet(test_model.net.Proto().name)
    test_accuracy[i] = workspace.FetchBlob('accuracy')
# After the execution is done, let's plot the values.
pyplot.plot(test_accuracy, 'r')
pyplot.title('Acuracy over test batches.')
print('test_accuracy: %f' % test_accuracy.mean())
pyplot.show()

# construct the model to be exported
# the inputs/outputs of the model are manually specified.
pe_meta = pe.PredictorExportMeta(
    predict_net=deploy_model.net.Proto(),
    parameters=[str(b) for b in deploy_model.params],
    inputs=["data"],
    outputs=["softmax"],
)

# save the model to a file. Use minidb as the file format
pe.save_to_db("minidb", os.path.join(root_folder, "mnist_model.minidb"), pe_meta)
print("The deploy model is saved to: " + root_folder + "/mnist_model.minidb")

# we retrieve the last input data out and use it in our prediction test before we scratch the workspace
blob = workspace.FetchBlob("data")
pyplot.figure()
_ = visualize.NCHW.ShowMultiple(blob)

# reset the workspace, to make sure the model is actually loaded
workspace.ResetWorkspace(root_folder)

# verify that all blobs are destroyed.
print("The blobs in the workspace after reset: {}".format(workspace.Blobs()))

# load the predict net
predict_net = pe.prepare_prediction_net(os.path.join(root_folder, "mnist_model.minidb"), "minidb")

# verify that blobs are loaded back
print("The blobs in the workspace after loading the model: {}".format(workspace.Blobs()))

# feed the previously saved data to the loaded model
workspace.FeedBlob("data", blob)

# predict
workspace.RunNetOnce(predict_net)
softmax = workspace.FetchBlob("softmax")

# the first letter should be predicted correctly
pyplot.figure()
_ = pyplot.plot(softmax[0], 'ro')
pyplot.title('Prediction for the first image')

pyplot.show()


from functools import lru_cache
from deap import creator, base, tools, algorithms
import random
import numpy as np
import os
import logging
import dask.dataframe as dd
from csci_final_project.dataloader import filehandler

creator.create("FitnessMax", base.Fitness, weights=(1.0,))
creator.create("Individual", list, fitness=creator.FitnessMax)

class GeneticAlgorithm:
    """
    The Genetic algorithm optimizer
    """

    def __init__(self, filename, filepath, target_name="pnl"):
        self.data = filehandler.loadCSV(filename, filepath)
        self.target_name = target_name
        self.results = []
        self.filename = filename
        self.filepath = filepath

    def _calculate_metrics(self, data):
        """The objective function metrics"""
        if len(data) > 0:
            return data[self.target_name].sum().compute()
        return 0

    @lru_cache(maxsize=100000)
    def _ga_optimize(self, parameter_values: tuple):
        """
        Here I use lru cache to memorize the parameter combinations that have searched before
        """
        ind = dict(parameter_values)
        data_filtered = dd.from_pandas(
            self.data, npartitions=max(1, os.cpu_count() - 1)
        )
        for col, val in ind.items():
            lower = val[0]
            upper = val[1]
            data_filtered = data_filtered.loc[
                (data_filtered[col] > lower) & (data_filtered[col] <= upper)
            ]

        return (self._calculate_metrics(data_filtered),)

    def _evaluate(self, parameter_values: list):
        return self._ga_optimize(tuple(parameter_values))

    def run_optimization(self, optimization_setting, population_size=100, ngen_size=30):
        """
        The entry point to run optimization
        """  # Get optimization setting and target
        settings = optimization_setting.generate_setting_ga()
        self.target_name = optimization_setting.target_name

        # Define parameter generation function
        def generate_parameter():
            return random.choice(settings)

        def mutate_individual(individual, indpb):
            size = len(individual)
            paramlist = generate_parameter()
            for i in range(size):
                if random.random() < indpb:
                    individual[i] = paramlist[i]
            return (individual,)

        # Set up genetic algorithem
        toolbox = base.Toolbox()
        toolbox.register(
            "individual", tools.initIterate, creator.Individual, generate_parameter
        )
        toolbox.register("population", tools.initRepeat, list, toolbox.individual)
        toolbox.register("mate", tools.cxTwoPoint)
        toolbox.register("mutate", mutate_individual, indpb=1)
        toolbox.register("evaluate", self._evaluate)
        toolbox.register("select", tools.selNSGA2)

        total_size = len(settings)
        pop_size = population_size  # number of individuals in each generation
        lambda_ = pop_size  # number of children to produce at each generation
        mu = int(
            pop_size * 0.8
        )  # number of individuals to select for the next generation

        cxpb = 0.95
        mutpb = 1 - cxpb
        ngen = ngen_size

        pop = toolbox.population(pop_size)
        hof = tools.ParetoFront()

        stats = tools.Statistics(lambda ind: ind.fitness.values)
        # np.set_printoptions(suppress=True)
        stats.register("mean", np.mean, axis=0)
        stats.register("std", np.std, axis=0)
        stats.register("min", np.min, axis=0)
        stats.register("max", np.max, axis=0)

        # Run ga optimization
        logging.info(f"total size：{total_size}")
        logging.info(f"population size：{pop_size}")
        logging.info(f"individuals for the next generation：{mu}")
        logging.info(f"number of generation：{ngen}")
        logging.info(f"probability of mating two individuals：{cxpb:.0%}")
        logging.info(f"probability of mutating an individual：{mutpb:.0%}")

        algorithms.eaMuPlusLambda(
            pop, toolbox, mu, lambda_, cxpb, mutpb, ngen, stats, halloffame=hof
        )

        # Return result list
        results = []

        for parameter_values in hof:
            setting = dict(parameter_values)
            target_value = self._evaluate(parameter_values)
            results.append((setting, target_value, {}))

        self.results = results
        logging.info(f"Best signal combination {results[0]} ")
        return results

    def output_data_best_signal(self, output_filename):
        """
        Output optimized data to csv
        """
        data_filtered = self.data.copy()
        for col, val in self.results[0][0].items():
            lower = val[0]
            upper = val[1]
            data_filtered = data_filtered.loc[
                (data_filtered[col] > lower) & (data_filtered[col] <= upper)
            ]
        filehandler.saveCSV(data_filtered, output_filename, self.filepath)

# from csci_final_project.optimizer.setting import OptimizationSetting
# setting = OptimizationSetting()
# setting.set_target('pnl')
# setting.add_parameter('signal_1', start=0, end=1, step=0.5)
# setting.add_parameter('signal_2', start=0, end=1, step=0.5)
#
# ga = GeneticAlgorithm(filename='sample.csv', filepath='/Users/xiaotan/Course/2020fa-csci_final_project-tanxiao64/data/')
# print(ga.run_optimization(setting))
# ga.output_data_best_signal('best.csv')


import re
from bs4 import BeautifulSoup
import sys
sys.path.append("..")
from crawler import DBHelper,CrawlerHelper
from crawler.shop import MGStage
#https://www.prestige-av.com/goods/goods_list.php?mode=&mid=&sort=near&word=&count=100&page=135
pcookie={'age_auth':'1'}
def spider_goodlist():
    baseurl='https://www.prestige-av.com/goods/goods_list.php?mode=&mid=&sort=near&word=&count=100&page={pageindex}'
    pageindex=1
    while True:
        print(f'page:{pageindex}')
        goodlisturl=baseurl.replace('{pageindex}',str(pageindex))
        html=CrawlerHelper.get_requests(goodlisturl,cookies=pcookie).text
        bs = BeautifulSoup(html, "html.parser")
        links=bs.find_all('a',href=re.compile('https://www.prestige-av.com/goods/goods_detail.php\?sku='))

        for link in links:
            url=link['href']
            dvdid=url[55:]
            title=link.img.get_text()

            dvdidreal=dvdid
            if dvdidreal.startswith('gooe-'):
                continue
            if dvdidreal.startswith('poh-'):
                continue
            elif dvdidreal.startswith('ctkt'):
                dvdidreal = dvdidreal.replace('ctkt', '', 1)
            elif dvdidreal.startswith('tkt'):
                dvdidreal = dvdidreal.replace('tkt', '', 1)  # GOOE
            elif dvdidreal.startswith('gooe') :
                    dvdidreal = dvdidreal.replace('gooe', '', 1)
            elif dvdidreal.startswith('sha'):
                    dvdidreal = dvdidreal.replace('sha', '', 1)
            elif dvdidreal.startswith('mgs'):
                    dvdidreal = dvdidreal.replace('mgs', '', 1)
            if DBHelper.check_dvdid_exist(dvdidreal.upper()):
                #print(f'{dvdid} existed')
                pass
            else:
                print(url)
                print(f'{dvdid} not found')
                crawler_goods_detail(dvdid)

        pageindex+=1
        if len(re.findall('次へ',html))==0:
            break
def crawler_goods_detail(dvdid):
    url=f'https://www.prestige-av.com/goods/goods_detail.php?sku={dvdid}'
    html = CrawlerHelper.get_requests(url, cookies=pcookie).text
    bs = BeautifulSoup(html, "html.parser")

    code=dvdid.upper()
    if code.startswith('CTKT'):
        code=code.replace('CTKT', '', 1)
    elif code.startswith('TKT'):
        code=code.replace('TKT', '', 1)#GOOE
    elif code.startswith('GOOE'):
        code = code.replace('GOOE', '', 1)
    elif code.startswith('SHA'):
        code = code.replace('SHA', '', 1)
    cid = code
    piccode = code

    title=bs.title.get_text().split('|')[0].strip()
    bs_detail=bs.find('div',class_='product_detail_layout_01')

    has_mgslink = False
    mgslink=bs_detail.find('a', href=re.compile(f'https://www.mgstage.com/product/product_detail/'))
    if mgslink:
        MGStage.spider_moviePage(mgslink['href'])
        return

    #
収録時間：
[\s]*
(\d*)min

    length=None
    length=re.findall('
収録時間：
[\s]*
(\d*)min
',html)
    if len(length) > 0:
        length=length[0]
    rdate=None
    rdatetag=bs_detail.find('a',href=re.compile('/goods/goods_list.php\?mode=date&mid='))
    if rdatetag:
        rdate=rdatetag.get_text().strip()
    studio=None
    studiotag = bs_detail.find('a', href=re.compile('/goods/goods_list.php\?mode=maker&mid='))
    if studiotag:
        studio = studiotag.get_text().strip()
    label = None
    labeltag = bs_detail.find('a', href=re.compile('/goods/goods_list.php\?mode=label&mid='))
    if labeltag:
        label = labeltag.get_text().strip()
    series = None
    seriestag = bs_detail.find('a', href=re.compile('/goods/goods_list.php\?mode=series&mid='))
    if seriestag:
        series = seriestag.get_text().strip()
    if series == '写真集':
        return
    actslist = []
    actstag=bs_detail.find_all('a',href=re.compile('/goods/goods_list.php\?mode=actress&mid='))
    for acttag in actstag:

        actslist.append(acttag.get_text().strip().replace(' ',''))
    genrelist = []
    genrestag = bs_detail.find_all('a', href=re.compile('/goods/goods_list.php\?mode=genre&mid='))
    for genretag in genrestag:
        genrename=genretag.get_text().strip()
        if genrename=='グッズ':
            return
        genrelist.append(genretag.get_text().strip())
    director=None
    piccount=len(bs.find_all('img', alt=title))
    print(f'has_mgslink:{has_mgslink}')
    print(f"cid:{cid}")
    print(f'code:{code}')
    print(f'title:{title}')
    print(f'length:{length}')
    print(f'rdate:{rdate}')
    print(f'director:{director}')
    print(f'studio:{studio}')
    print(f'label:{label}')
    print(f'series:{series}')
    print(f'pic:{piccode} count:{piccount}')
    print(actslist)
    print(genrelist)

    DBHelper.save_movie(code=code, cid=cid, title=title, length=length, rdate=rdate,
                        director=director, studio=studio, label=label, series=series,
                        piccode=piccode, piccount=piccount, source=1001,
                        actresslist=actslist, genrelist=genrelist)

if __name__ == '__main__':
    #print(len('https://www.prestige-av.com/goods/goods_detail.php?sku='))
    spider_goodlist()


from selenium.webdriver.common.by import By
from pages.base_page import BasePage
import allure

class MainPageScooter(BasePage):

    def __init__(self, driver):
        self.driver = driver

    logo_yandex = [By.XPATH, './/img[@alt = "Yandex"]//ancestor::a']
    page_yandex = [By.XPATH, './/button[text() ="Найти"]']

    @allure.step('Получаем ответ на вопрос в блоке "Вопросы о важном"')
    def get_answer_to_question(self, question, answer):
        self.wait_load_any_element(locator=question)
        self.driver.find_element(*question).click()
        self.wait_load_any_element(locator=answer)
        return self.driver.find_element(*answer).text

    @allure.step('Кликаем по логотипу "Яндекса"')
    def click_logo_yandex(self):
        self.wait_load_any_element(locator=self.logo_yandex)
        self.driver.find_element(*self.logo_yandex).click()
        self.switch_to_windows()
        self.wait_load_any_element(locator=self.page_yandex)


# imports

import os, cv2, itertools, re

import keras

from keras.utils.np_utils import to_categorical

from keras.datasets import mnist

from keras.layers import Convolution2D, Dropout

from keras.models import Sequential

from keras import layers

from keras.layers import Input, Add, Dense, Activation, ZeroPadding2D, BatchNormalization, Flatten, Conv2D, AveragePooling2D, MaxPooling2D

from keras.models import Model

import numpy as np

import pandas as pd

import seaborn as sns

import matplotlib.pyplot as plt

from sklearn.metrics import confusion_matrix

import random as rnd

from keras import backend as K

from keras.utils import np_utils

from keras.callbacks import CSVLogger

from keras.models import load_model

from keras.utils import np_utils as npu

from keras.preprocessing.image import ImageDataGenerator

from sklearn.utils import shuffle

import sklearn

from sklearn.model_selection import train_test_split

# function to return the most frequent element in a list (for good/bad predictions)

def most_frequent(listt):

    return max(set(listt), key = listt.count)

# the mnist model

def mnist_model():

    model = Sequential()

    model.add(Convolution2D(32, (3, 3), padding='same', input_shape=(28, 28, 1)))

    model.add(BatchNormalization())

    model.add(Activation('relu'))

    model.add(MaxPooling2D(pool_size=(2, 2), strides=None, padding='same'))

    model.add(Dropout(0.25))

    model.add(Convolution2D(64, (3, 3), padding='same'))

    model.add(BatchNormalization())

    model.add(Activation('relu'))

    model.add(MaxPooling2D(pool_size=(2, 2), strides=None, padding='same'))

    model.add(Dropout(0.25))

    model.add(Flatten())

    model.add(Dense(128))

    model.add(BatchNormalization())

    model.add(Activation('relu'))

    model.add(Dropout(0.25))

    model.add(Dense(10))

    model.add(Activation('softmax'))

    return model

# the cats vs dogs convolutional model

def animals_model():

    model = Sequential()

    model.add(Convolution2D(32, (3,3), padding='same', input_shape=(80, 80, 1)))

    model.add(BatchNormalization())

    model.add(Activation('relu'))

    model.add(MaxPooling2D(pool_size=(2, 2)))

    model.add(Convolution2D(64, (3,3), padding='same'))

    model.add(BatchNormalization())

    model.add(Activation('relu'))

    model.add(MaxPooling2D(pool_size=(2, 2)))

    model.add(Dropout(0.4))

    model.add(Convolution2D(128, (3,3), padding='same'))

    model.add(BatchNormalization())

    model.add(Activation('relu'))

    model.add(MaxPooling2D(pool_size=(2, 2)))

    model.add(Dropout(0.4))

    model.add(Convolution2D(256, (3,3), padding='same'))

    model.add(BatchNormalization())

    model.add(Activation('relu'))

    model.add(MaxPooling2D(pool_size=(2, 2)))

    model.add(Dropout(0.4))

    model.add(Convolution2D(512, (3,3), padding='same'))

    model.add(BatchNormalization())

    model.add(Activation('relu'))

    model.add(MaxPooling2D(pool_size=(2, 2)))

    model.add(Dropout(0.4))

    model.add(Flatten())

    model.add(Dropout(0.4))

    model.add(Dense(120))

    model.add(Activation('relu'))

    model.add(Dense(2))

    model.add(Activation('softmax'))

    return model
# setting the batch size for each epoch, number of classes (2), number of epochs

BATCH_SIZE = 64

num_classes = 2

NUM_EPOCHS = 50

# array of labels for plotting confusion matrix and badly classified images

animal_labels = ["cat", "dog"]

# paths to the train and test data

TRAIN_DIR = '../input/dogs-vs-cats-redux-kernels-edition/train/'

TEST_DIR = '../input/dogs-vs-cats-redux-kernels-edition/test/'

# data preparation

X = []

y = []

convert = lambda category : int(category == 'dog')

# importing and processing training data

def create_test_data(path):

    for p in os.listdir(path):

        if(os.path.join(path,p) != '../input/dogs-vs-cats-redux-kernels-edition/train/train'):

            category = p.split(".")[0]

            category = convert(category)

            # conversion to grayscale

            img_array = cv2.imread(os.path.join(path,p),cv2.IMREAD_GRAYSCALE)

            # resize all images to 80x80 pixels

            new_img_array = cv2.resize(img_array, dsize=(80, 80))

            X.append(new_img_array)

            y.append(category)

create_test_data(TRAIN_DIR)

X = np.array(X).reshape(-1, 80,80,1)

y = np.array(y)

# normalizing image data

X = X/255.0

# validation and training split

X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=1)

# importing the test data

X_test = []

def create_test1_data(path):

    for p in os.listdir(path):

        if(os.path.join(path,p) != '../input/dogs-vs-cats-redux-kernels-edition/test/test'):

            img_array = cv2.imread(os.path.join(path,p),cv2.IMREAD_GRAYSCALE)

            new_img_array = cv2.resize(img_array, dsize=(80, 80))

            X_test.append(new_img_array)

create_test1_data(TEST_DIR)

X_test = np.array(X_test).reshape(-1,80,80,1)

X_test = X_test/255

# one hot encoding of vector labels: 0 - cat, 1- dog

y_train = to_categorical(y_train)

y_val = to_categorical(y_val)

print(y_val.shape)

# initializing conv model

animal_cnn = animals_model()

# compiling model

animal_cnn.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

# imageDataGenerators for data augmentation

train_datagen = ImageDataGenerator(

    shear_range=0.2,

    zoom_range=0.2,

    horizontal_flip=True)

val_datagen = ImageDataGenerator(

    shear_range=0.2,

    zoom_range=0.2,

    horizontal_flip=True)

train_generator = train_datagen.flow(X_train, y_train, batch_size=BATCH_SIZE)

validation_generator = val_datagen.flow(X_val, y_val, batch_size=BATCH_SIZE)

# training the network

train_history = animal_cnn.fit_generator(

    train_generator,

    steps_per_epoch=X_train.shape[0] // BATCH_SIZE,

    epochs=NUM_EPOCHS,

    validation_data=validation_generator,

    verbose=0,

    validation_steps=X_val.shape[0] // BATCH_SIZE

)

# saving the network

animal_cnn.save('dogs_cats_model.h5')

# testing network performance

score = animal_cnn.evaluate(X_val, y_val, verbose=1)

print('Dogs vs Cats Evaluation Loss: ', score[0])

print('Dogs vs Cats Evaluation Accuracy: ', score[1])

# Print Model Stats

print('Training accuracy')

print(max(train_history.history['accuracy']))

print('Validation accuracy')

print(max(train_history.history['val_accuracy']))

# plotting training/validation accuracy/loss evolution

plt.figure()

plt.plot(train_history.history['accuracy'], color='C0', label='Training acc')

plt.title('Dogs vs Cats CNN Training Accuracy')

plt.ylabel('Accuracy')

plt.xlabel('Epoch')

plt.legend(loc='lower right')

plt.show()

plt.figure()

plt.plot(train_history.history['loss'], color='C0', label='Training loss')

plt.title('Dogs vs Cats CNN Training Loss')

plt.ylabel('Loss')

plt.xlabel('Epoch')

plt.legend(loc='upper right')

plt.show()

plt.figure()

plt.plot(train_history.history['val_accuracy'], color='C0', label='Validation acc')

plt.title('Dogs vs Cats CNN Validation Accuracy')

plt.ylabel('Accuracy')

plt.xlabel('Epoch')

plt.legend(loc='lower right')

plt.show()

plt.figure()

plt.plot(train_history.history['val_loss'], color='C0', label='Validation loss')

plt.title('Dogs vs Cats CNN Validation Loss')

plt.ylabel('Loss')

plt.xlabel('Epoch')

plt.legend(loc='upper right')

plt.show()

# getting predictions for the test data for the confusion matrix

y_pred = animal_cnn.predict(X_val)

y_pred = np.argmax(y_pred,axis=1)

y_true = np.asarray([np.argmax(i) for i in y_val])

print(y_pred.shape)

print(y_true.shape)

# generating confusion matrix

cm = confusion_matrix(y_true, y_pred)

print(cm)

cm_norm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]

# plotting confusion matrix

sns.set(font_scale=1.5)

fig, ax = plt.subplots(figsize=(10,10))

ax = sns.heatmap(

    cm_norm, annot=True, linewidths=0, square=False, cmap='Greens',

    yticklabels=animal_labels, xticklabels=animal_labels,

    vmin=0, vmax=np.max(cm_norm), fmt='.2f',

    annot_kws={'size': 20}

)

ax.set(xlabel='Predicted Label', ylabel='Actual Label', title='Dogs vs Cats CNN Confusion Matrix')

plt.show()

# plotting predictions for the first 10 test images

for x in range(0, 10):

    original_test_image = X_test[x].reshape(1,80, 80, 1)

    prediction = animal_cnn.predict(original_test_image)

    prediction_text = ""

    pred_val = max(prediction[0])

    if prediction[0][0] == pred_val:

        prediction_text = "cat"

    else:

        prediction_text = "dog"

    plt.xticks([])

    plt.yticks([])

    plt.imshow(X_test[x, :, :, 0], cmap='gray')

    title_text = "Predicted: " + prediction_text

    pic_name = "prediction" + str(x) + ".jpg"

    plt.title(title_text, fontsize=20)

    plt.savefig(pic_name)

    plt.show()


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit

df = pd.read_csv('Regression\china_gdp.csv')
x_data, y_data = (df["Year"].values, df["Value"].values)
xdata = x_data / max(x_data)
ydata = y_data / max(y_data)
'''plt.plot(df[['Year']], df[['Value']], color='blue')
plt.show()'''

# We're using a logistic function as it represents the trends in the data well

def sigmoid(x, Beta_1, Beta_2):
    y = 1 / (1 + np.exp(-Beta_1 * (x - Beta_2)))
    return y

beta_1 = 0.10
beta_2 = 1990.0

Y_pred = sigmoid(x_data, beta_1, beta_2)

# plot initial prediction against datapoints
'''plt.plot(x_data, Y_pred*15000000000000.)
plt.plot(x_data, y_data, 'ro')
plt.show()'''

popt, pcov = curve_fit(sigmoid, xdata, ydata)
print('beta1 = {} beta2 = {}'.format(popt[0], popt[1]))

# Visualize the fit/model
y = sigmoid(xdata, popt[0], popt[1])
plt.plot(xdata, ydata, 'ro', label='data')
plt.plot(xdata, y, linewidth=3.0, label='fit')
plt.legend(loc='best')
plt.ylabel('GDP')
plt.xlabel('Year')
plt.show()


import os
import sys
from typing import List

class Crawler:
    """
    A Crawler designed to traverse a file tree and print the files contained in it.
    """

    def __init__(self, target: str, extension_white_list: List[str] = [], abs_path: bool = False):
        """construct the File Crawler

        Arguments:
            target {str} -- the target directory for encryptioin

        Keyword Arguments:
            extension_white_list {List[str]} -- the list of file extensions to encrypt (default: {[]})
            abs_path {bool} -- whether or not to return the absolute path (default: {False})
        """

        self.target_directory: str = target
        self.files: List[str] = []
        self.file_extensions: str = extension_white_list
        self.abs_path: bool = abs_path

    def find_root(self) -> bool:
        """
        Returns whether or not the target directory is a directory, and finds the absolute path
        """
        return os.path.isdir(self.target_directory)

    def walk_tree(self) -> None:
        """
        Walks the given tree, and stores the individual file names encountered in the target directory
        """
        for root, subdirs, files in os.walk(self.target_directory):
            for file_name in files:
                for file_extension in self.file_extensions:
                    if file_name.endswith(file_extension):
                        if self.abs_path:
                            self.files.append(os.path.abspath(
                                root + os.sep + file_name))
                        else:
                            self.files.append(root + os.sep + file_name)

    def get_files(self) -> List[str]:
        """
        Get all the files that have been found by the Crawler

        Return:
            A list of strings, representing the file names of each file inside the target directory,
            with the given file extension.
        """
        return self.files


from typing import List

class Transaction():
    def __init__(self, id: int):
        self.id: int = id
        self.write_set: List[str] = []
        self.read_set: List[str] = []
        self.start_timestamp: int = 10e10
        self.validation_timestamp: int = 10e10
        self.finish_timestamp: int = 10e10

    def write(self, timestamp: int, data_item: str):
        if (len(self.write_set) == 0):
            self.validation_timestamp = timestamp

        if (len(self.read_set) == 0):
            self.start_timestamp = timestamp

        if (data_item not in self.write_set):
            print(f'Transaction {self.id} writes data item {data_item}')
            print(f'Current write set = {self.write_set}')
            self.write_set.append(data_item)

        self.start_timestamp = min(self.start_timestamp, timestamp)
        return True

    def read(self, timestamp: int, data_item: str):
        if (data_item not in self.read_set):
            self.read_set.append(data_item)
        print(f'Transaction {self.id} reads data item {data_item}')
        print(f'Current read set = {self.read_set}')

        self.start_timestamp = min(self.start_timestamp, timestamp)
        return True

    def commit(self, timestamp: int, transaction_arr: List['Transaction']):
        print()
        self.finish_timestamp = timestamp
        print(f'Transaction {self.id} commits, updating finish timestamp to {timestamp}')

        fail = True

        if (len(transaction_arr) == 1 and transaction_arr[0].id == self.id):
            fail = False

        for transaction in transaction_arr:
            if (transaction.id == self.id):
                fail = False
                continue

            print(f'Comparing with transaction {transaction.id}')

            if (transaction.start_timestamp >= self.start_timestamp):
                print(f'Compared transaction starts after this transaction, no conflict')
                fail = False
                continue

            # cek kondisi pertama, apakah waktu finish transaksi lain < waktu transaksi sendiri
            if (transaction.finish_timestamp < self.start_timestamp):
                print(f'Compared transaction finishes before current transactionbegins, can be committed')
                fail = False
                continue
            else:
                print(f'Compared transaction finishes after current transaction begins, needs further examination')

            # cek kondisi kedua, apakah waktu finish transaksi lain di antara waktu start dan validasi
            # dan yang dibaca transaksi ini tidak ditulis transaksi lain
            if (fail and self.start_timestamp < transaction.finish_timestamp < self.validation_timestamp):
                print(f'Compared transaction finishes between current transaction start and validation')
                if (set(self.read_set).isdisjoint(transaction.write_set)):
                    print(f'Disjoint between read set and write set of compared transaction, can be committed')
                    fail = False
                    continue
                else:
                    print(f'Intersect between read set and write set of compared transaction, cannot be committed')
            else:
                print(f'Compared transaction doesn\'t finish between current transaction start timestamp and validation timestamp, cannot be committed')

            if (fail):
                break

        if (not fail):
            print(f'Successfully committed transaction {self.id}!')
        else:
            print(f'Failed to commit transaction {self.id}!')
        print()
        return not fail

    def __str__(self):
        return f'T{self.id} ({self.read_set} {self.write_set})'


"""
2658. Maximum Number of Fish in a Grid
You are given a 0-indexed 2D matrix grid of size m x n, where (r, c) represents:

A land cell if grid[r][c] = 0, or
A water cell containing grid[r][c] fish, if grid[r][c] > 0.
A fisher can start at any water cell (r, c) and can do the following operations any number of times:

Catch all the fish at cell (r, c), or
Move to any adjacent water cell.
Return the maximum number of fish the fisher can catch if he chooses his starting cell optimally, or 0 if no water cell exists.

An adjacent cell of the cell (r, c), is one of the cells (r, c + 1), (r, c - 1), (r + 1, c) or (r - 1, c) if it exists.

Example1:
Input: grid = [[0,2,1,0],[4,0,0,3],[1,0,0,4],[0,3,2,0]]
Output: 7
Explanation: The fisher can start at cell (1,3) and collect 3 fish, then move to cell (2,3) and collect 4 fish.

Example2:
Input: grid = [[1,0,0,0],[0,0,0,0],[0,0,0,0],[0,0,0,1]]
Output: 1
Explanation: The fisher can start at cells (0,0) or (3,3) and collect a single fish.

Constraints:
m == grid.length
m == grid[i].length
1 <= m, n <= 10
0 <= grid[i][j] <= 10
"""

"""
Note:
1. UnionFind: O(mn) time | O(mn) space - where m is the length of row, n is the length of col
"""

import unittest
from typing import List
class UnionFind:
    def __init__(self, grid):
        rows, cols = len(grid), len(grid[0])
        total = rows * cols
        self.parents = [i for i in range(total)]
        self.ranks = [1 for _ in range(total)]
        self.counts = [grid[i//cols][i % cols] for i in range(total)]

    def find(self, u):
        if u != self.parents[u]:
            self.parents[u] = self.find(self.parents[u])
        return self.parents[u]

    def union(self, u, v):
        pu, pv = self.find(u), self.find(v)
        if pu == pv:
            return max(self.counts[pu], self.counts[pv])

        if self.ranks[pu] < self.ranks[pv]:
            self.parents[pu] = pv
            self.counts[pv] += self.counts[pu]
        elif self.ranks[pv] < self.ranks[pu]:
            self.parents[pv] = pu
            self.counts[pu] += self.counts[pv]
        else:
            self.parents[pv] = pu
            self.counts[pu] += self.counts[pv]
            self.ranks[pu] += 1

        newCount = max(self.counts[pu], self.counts[pv])
        return newCount

class Solution:
    def findMaxFish(self, grid: List[List[int]]) -> int:
        maxFish = 0
        unionFind = UnionFind(grid)
        rows, cols = len(grid), len(grid[0])

        def convert(r, c):
            return r*cols + c

        for row in range(rows):
            for col in range(cols):
                if grid[row][col] > 0:
                    u = convert(row, col)
                    maxFish = max(maxFish, grid[row][col])
                    if row+1 < rows and grid[row+1][col] > 0:
                        vD = convert(row+1, col)
                        maxFish = max(maxFish, unionFind.union(u, vD))
                    if col+1 < cols and grid[row][col+1] > 0:
                        vR = convert(row, col+1)
                        maxFish = max(maxFish, unionFind.union(u, vR))
        return maxFish

# Unit Tests
funcs = [Solution().findMaxFish]

class TestFindMaxFish(unittest.TestCase):
    def testFindMaxFish1(self):
        for func in funcs:
            grid = [[0, 2, 1, 0], [4, 0, 0, 3], [1, 0, 0, 4], [0, 3, 2, 0]]
            self.assertEqual(func(grid=grid), 7)

    def testFindMaxFish2(self):
        for func in funcs:
            grid = [[1, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 1]]
            self.assertEqual(func(grid=grid), 1)

if __name__ == "__main__":
    unittest.main()


import pygame, os, random

# My modules #
import colors, fonts, paths

from ball import PongBall
from paddle import Paddle
from difinfo import DifficultyInfo as difficulties

# The main function where everything runs #
def main():
    def resetArena(pointReset):
        ball.position.x = screen_Width / 2 - ball.size
        ball.position.y = screen_Height / 2
        ball.velocity.x = ball_speed
        ball.velocity.y = random.randint(-1, 1)

        paddle_left.position.y = screen_Height / 2 - paddle_size_y / 2
        paddle_right.position.y = screen_Height / 2 - paddle_size_y / 2

        if pointReset:
            points[0] = 0
            points[1] = 0

    def hitPaddle():
        ball.velocity.x = ball.velocity.x - (ball.velocity.x * 2)
        ball.velocity.y = random.randint(-1 - selected_difficulty, 1 + selected_difficulty) + ball.velocity.y - (ball.velocity.y * 2)
        pygame.mixer.Sound.play(ball_hit)

    # Starting pygame #
    pygame.init()

    # Window Settings #
    screen_Width, screen_Height, max_FrameRate = 640, 480, 30
    game_Title, bg_Color, icon = "Pong!", colors.BLACK, "pong_icon"

    # Setting up the screen #
    screen = pygame.display.set_mode((screen_Width, screen_Height))
    pygame.display.set_caption(game_Title)
    screen.fill(bg_Color)
    pygame.display.set_icon(pygame.image.load(paths.findPath(["images"], icon, "png")))

    # Main Variables #
    clock = pygame.time.Clock()
    points = [0, 0]
    ball_speed = 5

    ball = PongBall((320, 240), 20, colors.WHITE)

    paddle_speed = 10
    ai_speed = 9

    paddle_size_x = 20
    paddle_size_y = 100

    paddle_right = Paddle((screen_Width - paddle_size_x, screen_Height / 2 - paddle_size_y / 2), (paddle_size_x, paddle_size_y), colors.WHITE)
    paddle_left = Paddle((0, screen_Height / 2 - paddle_size_y / 2), (paddle_size_x, paddle_size_y), colors.WHITE)

    selected_difficulty = 1
    difficulty_string = "Fair"
    selection_screen = True

    # Fonts #
    pixel = pygame.font.Font(paths.findPath(["fonts"], fonts.pixel_font, "ttf"), 32)

    # Sounds #
    ball_hit = pygame.mixer.Sound(paths.findPath(["sounds"], "ball-hit", "wav"))
    score = pygame.mixer.Sound(paths.findPath(["sounds"], "score", "wav"))
    select = pygame.mixer.Sound(paths.findPath(["sounds"], "menu-select", "wav"))

    # Window Loop #
    running = True

    pygame.key.set_repeat(100)

    while running:
        # Caps the framerate #
        clock.tick(max_FrameRate)

        # Delta time for making things consistent #
        dt = clock.get_time()

        # Refreshes the screen with the background color #
        screen.fill(bg_Color)

        # Event handling #
        for event in pygame.event.get():
            # If you hold a key #
            if event.type == pygame.KEYDOWN:
                if not selection_screen:
                    # Paddle Movement #
                    if event.key == pygame.K_w:
                        paddle_right.position.y -= (paddle_speed / 10) * dt

                        if paddle_right.position.y < 0:
                            paddle_right.position.y = 0

                    if event.key == pygame.K_s:
                        paddle_right.position.y += (paddle_speed / 10) * dt

                        if paddle_right.position.y > screen_Height - paddle_right.size.y:
                            paddle_right.position.y = screen_Height - paddle_right.size.y

            if event.type == pygame.KEYUP:
                if selection_screen:
                    if event.key == pygame.K_RETURN:
                        resetArena(True)
                        selection_screen = False
                        pygame.mixer.Sound.play(select)

                    if event.key == pygame.K_UP:
                        selected_difficulty -= 1
                        pygame.mixer.Sound.play(select)

                    if event.key == pygame.K_DOWN:
                        selected_difficulty += 1
                        pygame.mixer.Sound.play(select)

                    if event.key == pygame.K_ESCAPE:
                        running = False
                else:
                    if event.key == pygame.K_ESCAPE:
                        resetArena(True)
                        selection_screen = True
                        pygame.mixer.Sound.play(select)

            # If you quit the program with the window's x button #
            if event.type == pygame.QUIT:
                # Stop running this loop #
                running = False

        # Draw UI before everything else so nothing goes behind it #

        # Difficulty Select #
        if selection_screen:
            if selected_difficulty < 0:
                selected_difficulty = len(difficulties) - 1
            elif selected_difficulty > len(difficulties) - 1:
                selected_difficulty = 0

            for x in range(0, len(difficulties)):
                string_val = difficulties[x][0]

                if x == selected_difficulty:
                    string_val = ">" + string_val + "<"

                    paddle_speed = difficulties[x][2][0]
                    ai_speed = difficulties[x][2][1]
                    difficulty_string = difficulties[x][0]

                text = pixel.render(string_val, True,  difficulties[x][1], bg_Color)
                text_rect = text.get_rect()
                text_rect.center = (630 // 2, (400 + (x * 100)) // 2)
                screen.blit(text, text_rect)

            splash = pixel.render("-- Difficulty Select --", True, colors.WHITE, bg_Color)
            splash_rect = splash.get_rect()
            splash_rect.center = (630 // 2, 200 // 2)
            screen.blit(splash, splash_rect)

            version_num = pixel.render("v0.0.1", True, colors.GREY, bg_Color)
            version_num_rect = version_num.get_rect()
            version_num_rect.center = (0, 0)
            version_num_rect.x = 0
            version_num_rect.y = screen_Height - version_num_rect.height
            screen.blit(version_num, version_num_rect)

        # Score Text #
        if not selection_screen:
            score_text = pixel.render(str(points[0]) + " - " + str(points[1]), True, colors.WHITE, bg_Color)
            score_text_rect = score_text.get_rect()
            score_text_rect.center = (630 // 2, 50 // 2)
            screen.blit(score_text, score_text_rect)

            selected = pixel.render("Difficulty: " + difficulty_string, True, colors.GREY, bg_Color)
            selected_rect = selected.get_rect()

            selected_rect.center = (0, 0)
            selected_rect.x = 0
            selected_rect.y = screen_Height - selected_rect.height

            screen.blit(selected, selected_rect)

        # Drawing Paddles To The Screen #
        if not selection_screen:
            pygame.draw.rect(screen, paddle_right.color, pygame.Rect(paddle_right.position.x, paddle_right.position.y, paddle_right.size.x, paddle_right.size.y))
            pygame.draw.rect(screen, paddle_left.color, pygame.Rect(paddle_left.position.x, paddle_left.position.y, paddle_left.size.x, paddle_left.size.y))

        # Draws the ball to the screen #
        if not selection_screen:
            pygame.draw.circle(screen, ball.color, ball.position, ball.size)

        # Adds the velocity of the ball to the ball's movement #
        if not selection_screen:
            ball.position.x += (ball.velocity.x / 10) * dt
            ball.position.y += (ball.velocity.y / 10) * dt

        # Ball Collision #
        # We use a rectangular hit box for simplicity #

        if not selection_screen:
            if paddle_right.is_touching(ball):
                hitPaddle()
            elif paddle_left.is_touching(ball):
                hitPaddle()

        # Ball Outside Checks #

        # Checks if the ball is out of bounds on the right side #
        if ball.position.x >= screen_Width + ball.size:
            points[0] += 1
            resetArena(False)
            pygame.mixer.Sound.play(score)
        # Checks if the ball is out of bounds on the right side #
        elif ball.position.x <= 0 - ball.size:
            points[1] += 1
            resetArena(False)
            pygame.mixer.Sound.play(score)

        # Ball Screen Check #
        if not selection_screen:
            if ball.position.y < 0 + ball.size:
                ball.position.y = 0 + ball.size
                ball.velocity.y = random.randint(-5, 5)

                if not ball.velocity.y <= 0:
                    pygame.mixer.Sound.play(ball_hit)
            elif ball.position.y > screen_Height - ball.size:
                ball.position.y = screen_Height - ball.size
                ball.velocity.y = random.randint(-5, 5)

                if not ball.velocity.y <= 0:
                    pygame.mixer.Sound.play(ball_hit)

        # Move Left Paddle #
        if not selection_screen:
            if ball.position.y > paddle_left.position.y:
                paddle_left.position.y += (ai_speed / 10) * dt

            if ball.position.y < paddle_left.position.y:
                paddle_left.position.y -= (ai_speed / 10) * dt

        # Left Paddle Screen Check #
        if paddle_left.position.y < 0:
            paddle_left.position.y = 0
        elif paddle_left.position.y > screen_Height - paddle_left.size.y:
            paddle_left.position.y = screen_Height - paddle_left.size.y

        # Updates the display #
        pygame.display.update()

    pygame.quit()

main()


import pygame
import pygame.gfxdraw
import random
from variables import *

class Plant:
    def __init__(self, x, y, energy):
        self.x = x
        self.y = y
        self.energy = energy

    def draw(self, screen, PLANT_COLOR, PLANT_COLOR2):
        radius = max(2, min(4, int(self.energy / 20)))
        pygame.gfxdraw.filled_circle(screen, int(self.x), int(self.y), radius, PLANT_COLOR)
        pygame.gfxdraw.aacircle(screen, int(self.x), int(self.y), radius, PLANT_COLOR)
        pygame.gfxdraw.aacircle(screen, int(self.x), int(self.y), radius - 2 , PLANT_COLOR2)
        pygame.gfxdraw.filled_circle(screen, int(self.x), int(self.y), radius - 2, PLANT_COLOR2)

    def generate_new_plant(plant):
        new_x = plant.x + random.randint(-200, 200)
        new_y = plant.y + random.randint(-200, 200)
        if 390 <= new_x <= SCREEN_WIDTH and SCREEN_HEIGHT - 590 <= new_y <= SCREEN_HEIGHT - 10:
            return Plant(new_x, new_y, plant.energy / 200)
        else:
            return None

    def grow(self):
        self.energy = min(self.energy + PLANT_GROWTH_RATE * self.energy, 100)


from enum import Enum

class Type(Enum):
    FC = 1
    ACTV = 2
    POOL = 3
    CONV = 4
    DEPTH = 5
    SEPAR = 6
    GEMM = 7
    LSTM = 8

class Buf(Enum):
    UBUF = 1
    WBUF = 2
    ACCQ = 3

class Op(Enum):
    LOAD_TILE = 1
    GEMM_OP = 2
    CONV_OP = 3
    VECTOR_OP = 4
    STORE_TILE = 5
    STORE_FAKE = 6
    NOP = 7

class Sched(Enum):
    FCFS = 1
    RRB = 2
    HPF = 3
    TOKEN = 4
    SJF = 5
    PREMA = 6

class Mecha(Enum):
    STATIC = 1
    DYNAMIC = 2