import torch
from torch import nn

__all__ = [
    'DotProductAttention'
,   'AdditiveAttention'
]

class Attention(nn.Module):
    '''base model for attention'''
    def __init__(self, *args, **kwargs) -> None:
        super().__init__()
        self.maskValue = -1e10

    def forward(self, *args, **kwargs):
        '''should be implemented in child class'''
        raise NotImplementedError

    def _shapeCheck(self, inputTensor:torch.Tensor, name:str, dim:int):
        '''check if inputTensor's dimension is equal to dim'''
        try:
            assert(len(inputTensor.shape) == dim)
        except AssertionError:
            raise TypeError(f'{name} must be a {dim}D tensor but got shape of {inputTensor.shape}')

    def _equalCheck(self, val0:any, name0:str, val1:any, name1:str):
        '''check if val0 is equal to val1'''
        if isinstance(val0, torch.Tensor):
            try:
                assert(torch.equal(val0, val1))
            except AssertionError:
                raise ValueError(f'{name0} should be the same as {name1}, but got {val0} and {val1}')
        else:
            try:
                assert(val0 == val1)
            except AssertionError:
                raise ValueError(f'{name0} should be the same as {name1}, but got {val0} and {val1}')

    def _mask(self, score, mask):
        '''
            mask is a 0-1 tensor with the same shape of score
            set score[where mask = 0] to self.maskValue(-1e10)
        '''
        self._equalCheck(score.shape, 'shape of score', mask.shape, 'shape of mask')
        return score.masked_fill(mask == 0, self.maskValue)

class DotProductAttention(Attention):
    '''dot product attention implementation'''
    def __init__(self, *args, **kwargs) -> None:
        super().__init__(args, kwargs)
        self.softmax = nn.Softmax(dim=-1)

    def forward(self, queries:torch.Tensor, keys:torch.Tensor, values:torch.Tensor, mask = None):
        '''
        input
            queries: (batchSize, n, dk) note that dimension of query must be same as key
            keys: (batchSize, m, dk)
            values: (batchSize, m, dv)
            mask: (batchSize, n, m)
        return
            attention output: (batchSize, n, dv) for n queries
        '''

        self._shapeCheck(queries, 'queries', dim=3)
        self._shapeCheck(keys, 'keys', dim=3)
        self._shapeCheck(values, 'values', dim=3)

        self._equalCheck(queries.shape[2], 'query\'s length', keys.shape[2], 'key\'s length')
        self._equalCheck(keys.shape[1], 'amount of keys', values.shape[1], 'amount of values')

        attentionScore = torch.bmm(queries, keys.transpose(1, 2)) / keys.shape[2]**0.5 # (batchSize, n, m)

        if mask is not None:
            attentionScore = self._mask(attentionScore, mask)

        attentionWeight = self.softmax(attentionScore)
        return torch.bmm(attentionWeight, values)

class AdditiveAttention(Attention):
    '''additive attention implementation'''
    def __init__(self, querySize:int, keySize:int, numHidden:int, **kwargs) -> None:
        super().__init__(querySize, keySize, numHidden, **kwargs)
        self.queryTransform = nn.Linear(querySize, numHidden) # transform query dim from querySize to numHidden
        self.keyTransform = nn.Linear(keySize, numHidden) # transform key dim from keySize to numHidden
        self.tanh = nn.Tanh()
        self.valueTransform = nn.Linear(numHidden, 1) # transform key-query coincidence to scalar for attention score
        self.softmax = nn.Softmax(dim=-1)

    def forward(self, queries:torch.Tensor, keys:torch.Tensor, values:torch.Tensor, mask = None):
        '''
        input
            queries: (batchSize, n, dq)
            keys: (batchSize, m, dk)
            values: (batchSize, m, dv)
        return
            attention output: (batchSize, n, dv) for n queries
        '''

        self._shapeCheck(queries, 'queries', dim=3)
        self._shapeCheck(keys, 'keys', dim=3)
        self._shapeCheck(values, 'values', dim=3)

        self._equalCheck(keys.shape[1], 'amount of keys', values.shape[1], 'amount of values')

        transformedQueries = self.queryTransform(queries) # (batchSize, n, h)
        transformedKeys = self.keyTransform(keys) # (batchSize, m, h)

        transformedQueries = transformedQueries.unsqueeze(2)  # (batchSize, n, 1, h)
        transformedKeys = transformedKeys.unsqueeze(1) # (batchSize, 1, m, h)

        coincidence = self.tanh(transformedKeys+transformedQueries) # (batchSize, n, m, h)

        attentionScore = self.valueTransform(coincidence) # (batchSize, n, m, 1)
        attentionScore = attentionScore.squeeze(-1) # (batchSize, n, m)

        if mask is not None:
            attentionScore = self._mask(attentionScore, mask)

        attentionWeight = self.softmax(attentionScore) # (batchSize, n, m)
        return torch.bmm(attentionWeight, values)

class MutiHeadAttention(Attention):
    '''muti-head attention implementation with dot product attention'''
    def __init__(self, querySize:int, keySize:int, valueSize:int, numHidden:int, numHeads:int, **kwargs) -> None:
        super().__init__(querySize, keySize, valueSize, numHidden, numHeads, **kwargs)
        self.numHeads = numHeads
        self.attention = DotProductAttention(querySize, keySize, valueSize, numHidden, **kwargs)
        self.queryTransform = nn.Linear(querySize, numHidden) # transform query dim from querySize to numHidden
        self.keyTransform = nn.Linear(keySize, numHidden) # transform key dim from keySize to numHidden
        self.valueTransform = nn.Linear(valueSize, numHidden) # transform value dim from valueSize to numHidden
        self.outputTransform = nn.Linear(numHidden, numHidden)

    def forward(self, queries:torch.Tensor, keys:torch.Tensor, values:torch.Tensor, mask = None):
        '''
        input
            queries: (batchSize, n, dq)
            keys: (batchSize, m, dk)
            values: (batchSize, m, dv)
            mask: (batchSize, n, m)
        return
            attention output: (batchSize, n, numHidden) for n queries
        '''
        self._shapeCheck(queries, 'queries', dim=3)
        self._shapeCheck(keys, 'keys', dim=3)
        self._shapeCheck(values, 'values', dim=3)

        self._equalCheck(keys.shape[1], 'amount of keys', values.shape[1], 'amount of values')

        transformedQueries = self.queryTransform(queries) # (batchSize, n, hidden)
        transformedKeys = self.keyTransform(keys) # (batchSize, m, hidden)
        transformedValues = self.valueTransform(values) # (batchSize, m, hidden)

        transformedQueries = self.split(transformedQueries) # (b*numHeads, n, hidden/numHeads)
        transformedKeys = self.split(transformedKeys) # (b*numHeads, m, hidden/numHeads)
        transformedValues = self.split(transformedValues) # (b*numHeads, m, hidden/numHeads)

        if mask is not None:
            # On axis 0, copy the first item (scalar or vector) for num_heads
            # times, then copy the next item, and so on
            mask = torch.repeat_interleave(mask, repeats=self.numHeads, dim=0)

        out = self.attention(transformedQueries, transformedKeys, transformedValues, mask)
        out = self.concat(out) # (b, n, hidden)
        return self.outputTransform(out) # (b, n, hidden)

    def split(self, X:torch.Tensor):
        '''
        input
            X: (b, n, d)
        return
            out: (b*numHeads, n, d/numHeads)
        '''
        try:
            assert(X.shape[2] % self.numHeads == 0)
        except AssertionError:
            raise ValueError(f'length of vector should be no. of heads\' multiple, but got {X.shape[2]} % numHeads != 0')

        X = X.reshape(X.shape[0], X.shape[1], self.numHeads, -1) # (b, n, numHeads, d/numHeads)
        X = X.transpose(1, 2) # (b, numHeads, n, d/numHeads)
        return X.reshape(X.shape[0]*X.shape[1], X.shape[2], X.shape[3])

    def concat(self, X:torch.Tensor):
        '''
        input
            X: (b*numHeads, n, d/numHeads)
        return
            out: (b, n, d)
        '''
        X = X.reshape(-1, self.numHeads, X.shape[1], X.shape[2])
        X = X.transpose(1, 2)
        return X.reshape(X.shape[0], X.shape[1], -1)

if __name__ == '__main__':
    attension = MutiHeadAttention(128, 512, 256, 128, 8)

    queries = torch.randn((10, 100, 128))
    keys = torch.randn((10, 1024, 512))
    values = torch.randn((10, 1024, 256))

    output = attension(queries, keys, values, torch.randint(0, 1, (queries.shape[0], queries.shape[1], keys.shape[1])))
    print(output.shape)


from flask import Flask, render_template, request, Response, redirect, url_for , session , jsonify , flash
from flask_bootstrap import Bootstrap

from object_detection import *
import object_detection

from flask_sqlalchemy import SQLAlchemy # import sqlalchemy
from database import db , Vehicle , DB_Manager

import webbrowser

from threading import Timer #Debug Autostart

application = Flask(__name__)
application.config.update(
    TESTING = True,
    SECRET_KEY = "password"
)
application.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///db/vehicle_db.sqlite3' # Config to use sqlalchemy
application.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False

db = SQLAlchemy(application)
db_manager = DB_Manager()

VIDEO = VideoStreaming(camera_src = 0)

@application.route('/')
def home():
    page_title = 'SecureV | Home'
    current_plate = ""
    current_owner = ""
    return render_template('index.html', TITLE=page_title , PLATE = current_plate, OWNER = current_owner)

@application.route('/update_plate' , methods=['POST'])
def updateplate():
    current_plate = object_detection.current_plate
    db_manager.get_db_data() # Not sure
    try:
        current_owner = db_manager.search_owner(current_plate)
    except:
        current_owner = "Not Verified"

    return jsonify('' , render_template('dynamic_plate.html', PLATE = current_plate , OWNER = current_owner))

@application.route('/update_gate' , methods=['POST'])
def updategate():
    # current_plate = object_detection.current_plate
    # current_g_status = False
    current_g_status = "Closed"
    return jsonify('' , render_template('dynamic_gate.html', GATE = current_g_status ))

@application.route('/video_feed')
def video_feed():
    '''
    Video streaming route.
    '''
    return Response(
        VIDEO.show(),
        mimetype='multipart/x-mixed-replace; boundary=frame'
    )

@application.route('/request_model_switch')
def request_model_switch():
    #TODO: On toggle OFF turn current-plate to XXX-XXXX
    # if VIDEO.detect == True:
    #     # current_plate = "XXX-XXXX"
    #     object_detection.current_plate = "XXX-XXXX"
    VIDEO.detect = not VIDEO.detect

    try:
        print("This is a return function from VideoStreaming class " + str(VIDEO.lblret))
    except:
        pass
    return "nothing"

@application.route("/database" , methods=["POST","GET"])
def data_mode():
    page_title = 'SecureV | Database Mode'
    db_man = DB_Manager()
    error = False
    error_message = ""

    try:
        if request.method =="POST":
            session.permanent = True
            car_to_delete = request.form["delete_plate_input"].upper()
            if car_to_delete:
                db_man.delete_car_and_all_entries(car_to_delete)
                flash(f"Deleted Vehicle:  [{car_to_delete}] and all Entries ", "info")

            else:
                error = True
                error_message = "Input Plate isn't registered or Input is empty"

            return redirect(url_for('data_mode'))

    except Exception as e:
        print(f"EXCEPTION AT /database route: {e}")
        error = True
        error_message = "Vehicle Plate is invalid or not existing"

    return render_template("data_mode.html" , db_data = db_man.db_data, TITLE=page_title , error = error , error_msg = error_message)

@application.route("/register", methods=["POST","GET"])
def register_mode():
    page_title = 'SecureV | Register Mode'
    error = False
    error_message = ""
    try:
        if request.method =="POST":
            session.permanent = True
            plate_input = request.form["plate_input"].upper()
            owner_input = request.form["owner_input"].title()
            if plate_input:
                current_registering = Vehicle(plate_num = plate_input , owner_name = owner_input)
                db.session.add(current_registering)
                db.session.commit()
                db_manager.get_db_data()
                flash(f"Successfully Registered [{plate_input}]", "info")
            else:
                error = True
                error_message = "Input Box Empty"
    except Exception as e:
        error = True
        error_message = "Error Input Box Value"
    return render_template("register_mode.html", TITLE=page_title , error = error , error_msg = error_message)

@application.route("/logs")
def log_mode():
    db_man = DB_Manager()
    page_title = 'SecureV | Log Mode'
    return render_template("log_mode.html", db_data = db_man.db_data_entries, TITLE=page_title)

def open_browser():
    ''' Debug autostartt'''
    webbrowser.open_new('http://127.0.0.1:2000/')

if __name__ == '__main__':
    # Timer(3, open_browser).start() # Auto open browser
    # db.create_all() # Create db when it doesnt exist
    application.run(port = 2000 , debug = True)


# -*- coding: utf-8 -*-
'''
Functions used for CLI argument handling
'''
from __future__ import absolute_import

# Import python libs
import re
import inspect

# Import salt libs
from salt.ext.six import string_types, integer_types
import salt.ext.six as six

#KWARG_REGEX = re.compile(r'^([^\d\W][\w.-]*)=(?!=)(.*)$', re.UNICODE)  # python 3
KWARG_REGEX = re.compile(r'^([^\d\W][\w.-]*)=(?!=)(.*)$')

def condition_input(args, kwargs):
    '''
    Return a single arg structure for the publisher to safely use
    '''
    ret = []
    for arg in args:
        if isinstance(arg, long):
            ret.append(str(arg))
        else:
            ret.append(arg)
    if isinstance(kwargs, dict) and kwargs:
        kw_ = {'__kwarg__': True}
        for key, val in six.iteritems(kwargs):
            kw_[key] = val
        return ret + [kw_]
    return ret

def parse_input(args, condition=True):
    '''
    Parse out the args and kwargs from a list of input values. Optionally,
    return the args and kwargs without passing them to condition_input().

    Don't pull args with key=val apart if it has a newline in it.
    '''
    _args = []
    _kwargs = {}
    for arg in args:
        if isinstance(arg, string_types):
            arg_name, arg_value = parse_kwarg(arg)
            if arg_name:
                _kwargs[arg_name] = yamlify_arg(arg_value)
            else:
                _args.append(yamlify_arg(arg))
        elif isinstance(arg, dict):
            # Yes, we're popping this key off and adding it back if
            # condition_input is called below, but this is the only way to
            # gracefully handle both CLI and API input.
            if arg.pop('__kwarg__', False) is True:
                _kwargs.update(arg)
            else:
                _args.append(arg)
        else:
            _args.append(arg)
    if condition:
        return condition_input(_args, _kwargs)
    return _args, _kwargs

def parse_kwarg(string_):
    '''
    Parses the string and looks for the following kwarg format:

    "{argument name}={argument value}"

    For example: "my_message=Hello world"

    Returns the kwarg name and value, or (None, None) if the regex was not
    matched.
    '''
    try:
        return KWARG_REGEX.match(string_).groups()
    except AttributeError:
        return None, None

def yamlify_arg(arg):
    '''
    yaml.safe_load the arg
    '''
    if not isinstance(arg, string_types):
        return arg

    if arg.strip() == '':
        # Because YAML loads empty strings as None, we return the original string
        # >>> import yaml
        # >>> yaml.load('') is None
        # True
        # >>> yaml.load('      ') is None
        # True
        return arg

    elif '_' in arg and all([x in '0123456789_' for x in arg.strip()]):
        return arg

    try:
        # Explicit late import to avoid circular import. DO NOT MOVE THIS.
        import salt.utils.yamlloader as yamlloader
        original_arg = arg
        if '#' in arg:
            # Only yamlify if it parses into a non-string type, to prevent
            # loss of content due to # as comment character
            parsed_arg = yamlloader.load(arg, Loader=yamlloader.SaltYamlSafeLoader)
            if isinstance(parsed_arg, string_types) or parsed_arg is None:
                return arg
            return parsed_arg
        if arg == 'None':
            arg = None
        else:
            arg = yamlloader.load(arg, Loader=yamlloader.SaltYamlSafeLoader)

        if isinstance(arg, dict):
            # dicts must be wrapped in curly braces
            if (isinstance(original_arg, string_types) and
                    not original_arg.startswith('{')):
                return original_arg
            else:
                return arg

        elif arg is None \
                or isinstance(arg, (list, float, integer_types, string_types)):
            # yaml.safe_load will load '|' as '', don't let it do that.
            if arg == '' and original_arg in ('|',):
                return original_arg
            # yaml.safe_load will treat '#' as a comment, so a value of '#'
            # will become None. Keep this value from being stomped as well.
            elif arg is None and original_arg.strip().startswith('#'):
                return original_arg
            else:
                return arg
        else:
            # we don't support this type
            return original_arg
    except Exception:
        # In case anything goes wrong...
        return original_arg

def get_function_argspec(func):
    '''
    A small wrapper around getargspec that also supports callable classes
    '''
    if not callable(func):
        raise TypeError('{0} is not a callable'.format(func))

    if inspect.isfunction(func):
        aspec = inspect.getargspec(func)
    elif inspect.ismethod(func):
        aspec = inspect.getargspec(func)
        del aspec.args[0]  # self
    elif isinstance(func, object):
        aspec = inspect.getargspec(func.__call__)
        del aspec.args[0]  # self
    else:
        raise TypeError('Cannot inspect argument list for {0!r}'.format(func))

    return aspec


from django.urls import path
from .views import (
    ArticulosAgregarView, ArticulosApiView, ArticulosDetalladoView, ArticulosEliminarView,ArticuloActualizarView)

urlpatterns = [
    path('api/', ArticulosApiView.as_view(), name="Api"),
    path('api/agregar/', ArticulosAgregarView.as_view(), name="create"),
    path('api/detallado/<pk>/', ArticulosDetalladoView.as_view(), name="detail"),
    path('api/eliminar/<pk>/', ArticulosEliminarView.as_view(), name="delete"),
    path('api/actualizar/<pk>/', ArticuloActualizarView.as_view(), name="update"),

]


import os
os.system('pip3 install lightgbm==2.1.2')
os.system('pip3 install hyperopt')

import pandas as pd
import pickle
import data_converter
import numpy as np
import scipy
from os.path import isfile
import random
import time
from sklearn.model_selection import train_test_split, StratifiedKFold
from sklearn.metrics import roc_auc_score
import sklearn
from multiprocessing import Pool

from preprocess import *
from boosting import *
from automl import *

module = ['read_data', 'preprocess', 'encode_cat', 'encode_mv', 'fit', 'predict']

class Model:
    def __init__(self, data_info, time_info):
        '''
        This constructor is supposed to initialize data members.
        Use triple quotes for function documentation.
        '''
        self.num_train_samples = 0
        self.num_feat = 1
        self.num_labels = 1

        self.is_trained = False
        self.batch_num = 0
        self.total_batch_num = 10

        self.batch_window_size = 3

        self.y_pred = None
        self.data_memory = []

        self.hyper_param = None

        self.missing_value_preprocess = MissingValuePreprocess(drop_ratio=0.9)

        self.overall_time = np.zeros(10, dtype='float')
        self.module_time = {}
        for m in module:
            self.module_time[m] = np.zeros(10, dtype='float')

    def fit(self, F, y, data_info, time_info):

        # time budget score
        if not self.is_trained:
            weight = [1., 1., 2., 4.]
            feat_weight = 0.0
            for i in range(4):
                feat_weight += weight[i] * data_info['loaded_feat_types'][i]
            self.budget_score = float(data_info['time_budget']) / (y.shape[0] * 10. / 1e6) / feat_weight
            self.time_manager = TimeManager(self.budget_score, self.batch_window_size)
            print ('budget score: %.2f' %(self.budget_score))

            self.use_mv = False
            if (data_info['loaded_feat_types'][3] != 0 and self.budget_score > 3.5):
                self.use_mv = True

        # read data
        if not self.is_trained:
            self.module_time['read_data'][self.batch_num] = time.time() - time_info[1]

        y = y.ravel()

        # preprocessing
        if not self.is_trained:

            module_start = time.time()

            F['numerical'] = self.missing_value_preprocess.fit_transform(F['numerical'], 'numerical', input_type='ndarray')
            F['CAT'] = self.missing_value_preprocess.fit_transform(F['CAT'], 'CAT', input_type='dataframe')
            if self.use_mv:
                F['MV'] = self.missing_value_preprocess.fit_transform(F['MV'], 'MV', input_type='dataframe')

            F['numerical'] = data_converter.replace_missing(F['numerical']).astype('float32')
            F['CAT'] = F['CAT'].fillna('-1')

            module_end = time.time()
            self.module_time['preprocess'][self.batch_num] = module_end - module_start

            self.F = F

        # store current batch of data
        if (len(self.data_memory) == self.batch_window_size):
            del self.data_memory[0]
        self.data_memory.append([self.F, y])

        self.batch_end_time = time.time()
        if self.is_trained:
            self.batch_start_time = self.next_batch_start_time
        else:
            self.batch_start_time = time_info[1]

        self.overall_time[self.batch_num] = self.batch_end_time - self.batch_start_time
        print ('overall time spent on batch %d: %.2f seconds' %(self.batch_num, self.overall_time[self.batch_num]))
        for m in module:
            t = self.module_time[m][self.batch_num]
            ratio = t / self.overall_time[self.batch_num]
            print ('%s: %.2f seconds, %.2f%%' %(m, t, ratio * 100.))
        if self.is_trained:
            print ('time spent ratio: %.2f%%' %(self.time_spent_ratio))

        self.fit_end_time = time.time()
        self.next_batch_start_time = time.time()

    def transferPredict(self, i, data_info, time_info):

        train_num = np.concatenate([self.data_memory[j][0]['numerical'] for j in range(i, len(self.data_memory))], axis=0)
        train_cat = pd.concat([self.data_memory[j][0]['CAT'] for j in range(i, len(self.data_memory))], axis=0, ignore_index=True, copy=False)
        label = np.concatenate([self.data_memory[j][1] for j in range(i, len(self.data_memory))])

        test_num = self.F['numerical']
        test_cat = self.F['CAT']

        module_start = time.time()

        # encode categorical feature
        all_cat = pd.concat([train_cat, test_cat], axis=0, ignore_index=True, copy=False)
        del train_cat, test_cat
        cat_columns = all_cat.columns.copy()

        with Pool(processes=2) as pool:
            cat_encode = pool.map(CATEncoder().fit_transform, [all_cat[[col]] for col in cat_columns])
            pool.close()
            pool.join()
        all_cat = pd.concat(cat_encode, axis=1, copy=False)
        del cat_encode

        train_cat, test_cat = all_cat.iloc[:label.shape[0], :].reset_index(drop=True), all_cat.iloc[label.shape[0]:, :].reset_index(drop=True)
        del all_cat

        module_end = time.time()
        self.module_time['encode_cat'][self.batch_num] = module_end - module_start

        train_num, test_num = pd.DataFrame(train_num), pd.DataFrame(test_num)
        train_feature = [train_num, train_cat]
        test_feature = [test_num, test_cat]

        # encode multi-value feature
        if self.use_mv:

            module_start = time.time()

            max_cat_num = 1000

            all_mv = pd.concat([self.data_memory[j][0]['MV'] for j in range(i, len(self.data_memory))], axis=0)
            all_mv = pd.concat([all_mv, self.F['MV']], axis=0)
            mv_columns = all_mv.columns.copy()

            with Pool(processes=2) as pool:
                mv_encode = pool.map(MVEncoder(max_cat_num=max_cat_num).fit_transform, [all_mv[col] for col in mv_columns])
                all_mv = pd.concat(mv_encode, axis=1)
                pool.close()
                pool.join()

            all_mv = all_mv.astype('int16')
            train_mv, test_mv = all_mv.iloc[:label.shape[0], :].reset_index(drop=True), all_mv.iloc[label.shape[0]:, :].reset_index(drop=True)
            del all_mv

            train_feature.append(train_mv)
            test_feature.append(test_mv)

            module_end = time.time()
            self.module_time['encode_mv'][self.batch_num] = module_end - module_start

        feature_new = pd.concat(train_feature, axis=1)
        for feat in train_feature:
            del feat
        del train_feature

        self.X_new = pd.concat(test_feature, axis=1)
        for feat in test_feature:
            del feat
        del test_feature

        weight = None

        # time spent on data/feature processing in current batch
        time_spent_invariant, time_spent_variant = 0., 0.
        for m in ['read_data', 'preprocess']:
            time_spent_invariant += self.module_time[m][self.batch_num]
        for m in ['encode_cat', 'encode_mv']:
            time_spent_variant += self.module_time[m][self.batch_num]

        # estimated time budget for each remaining batch
        time_left = data_info['time_budget'] - (self.predict_start_time - time_info[1])
        batch_left = self.total_batch_num - self.batch_num
        time_per_batch = time_left / batch_left

        # estimated time budget for training models
        ratio = float(self.batch_window_size + 1) / (min(self.batch_num, self.batch_window_size) + 1)
        time_for_model = time_per_batch - time_spent_invariant - time_spent_variant * ratio

        clf = GBM(category_cols=None, hyper_tune=False, hyper_param=self.hyper_param)

        if (self.batch_num == 1):
            # decide boosting iterations
            self.suggested_boost_round = self.time_manager.suggest_boosting_round(feature_new, label, time_for_model, self.batch_num)
            self.suggested_learning_rate, self.early_stop_round = clf.suggest_learning_rate(feature_new, label, self.suggested_boost_round)
            self.num_boost_round = self.early_stop_round
            print ('max boost round suggested by time manager: %d' %(self.suggested_boost_round))
            print ('early stop round: %d' %(self.early_stop_round))
            print ('suggested learning rate: %f' %(self.suggested_learning_rate))
        else:
            # adjust boosting round number according to time budget
            ratio = min(self.batch_window_size, self.batch_num) / (min(self.batch_window_size, self.batch_num) - 1)
            estimated_train_time = self.module_time['fit'][self.batch_num - 1] * ratio
            remaining_time = time_for_model - estimated_train_time
            if (remaining_time > time_for_model * 0.1 and self.num_boost_round < 2000):
                self.num_boost_round += 200
            elif (remaining_time < 0.):
                self.num_boost_round -= 100

        print ('training info of batch %d: ' %(self.batch_num))
        print ('feature number: %d' %(feature_new.shape[1]))
        print ('train sample size: %d' %(feature_new.shape[0]))
        print ('test sample size: %d' %(self.X_new.shape[0]))
        print ('num boost round: %d' %(self.num_boost_round))
        print ('learning rate: %f' %(self.suggested_learning_rate))

        # LGB
        module_start = time.time()

        updated_params = {'n_estimators': self.num_boost_round, 'learning_rate': self.suggested_learning_rate}
        self.hyper_param = clf.fit(feature_new, label, weight=weight, updated_params=updated_params)

        module_end = time.time()
        self.module_time['fit'][self.batch_num] = module_end - module_start

        module_start = time.time()

        y_pred = clf.predict(self.X_new)

        module_end = time.time()
        self.module_time['predict'][self.batch_num] = module_end - module_start

        self.time_spent_ratio = (time.time() - self.predict_start_time) / time_per_batch * 100.

        self.is_trained = True

        return y_pred

    def predict(self, F, data_info, time_info):
        '''
        This function should provide predictions of labels on (test) data.
        Here we just return random values...
        Make sure that the predicted values are in the correct format for the scoring
        metric. For example, binary classification problems often expect predictions
        in the form of a discriminant value (if the area under the ROC curve it the metric)
        rather that predictions of the class labels themselves.
        The function predict eventually can return probabilities or continuous values.
        '''
        self.batch_num += 1
        self.predict_start_time = time.time()
        self.module_time['read_data'][self.batch_num] = (self.predict_start_time - self.fit_end_time)

        module_start = time.time()

        F['numerical'] = self.missing_value_preprocess.transform(F['numerical'], 'numerical', input_type='ndarray')
        F['CAT'] = self.missing_value_preprocess.transform(F['CAT'], 'CAT', input_type='dataframe')
        if self.use_mv:
            F['MV'] = self.missing_value_preprocess.transform(F['MV'], 'MV', input_type='dataframe')

        F['numerical'] = data_converter.replace_missing(F['numerical']).astype('float32')
        F['CAT'] = F['CAT'].fillna('-1')

        module_end = time.time()
        self.module_time['preprocess'][self.batch_num] = module_end - module_start

        self.F = F

        self.y_pred = self.transferPredict(0, data_info, time_info)

        self.predict_end_time = time.time()
        return self.y_pred

    def save(self, path="./"):
        pickle.dump(self, open(path + '_model.pickle', "w"))

    def load(self, path="./"):
        modelfile = path + '_model.pickle'
        if isfile(modelfile):
            with open(modelfile) as f:
                self = pickle.load(f)
            print("Model reloaded from: " + modelfile)
        return self


#coding=utf-8
'''
#import urllib
#import urllib.request
#import urllib.parse
import requests

URL_IP = 'http://httpbin.org/ip'
URL_GET = 'http://httpbin.org/get'

def use_simple_requests():
    response = requests.get(URL_IP)
    print('>>>Response Headers:')
    print(response.headers)
    print('>>>Response  Body:')
    print(response.text)

def use_params_requests():
    params = {'param1':'hello','param2':'你好','param3':'test'}
    response = requests.get(URL_GET,params=params)
    print('>>>Response Headers:')
    print(response.headers)
    print('>>>Status Code:')
    print(response.status_code)
    print('>>>Status Code reason:')
    print(response.reason)
    print('>>>Response  Body:')
    #print(response.json)
    print(response.text)

if __name__ == '__main__':
    print ('>>>Use simple urllib2:')
    use_params_requests()

'''
'''
内置高阶函数
一：
def f(x):
    return x*x
print map(f, [1, 2, 3, 4, 5, 6, 7, 8, 9])

二：

例如，编写一个f函数，接收x和y，返回x和y的和：

def f(x, y):
    return x + y

调用 reduce(f, [1, 3, 5, 7, 9])时，reduce函数将做如下计算：

先计算头两个元素：f(1, 3)，结果为4；
再把结果和第3个元素计算：f(4, 5)，结果为9；
再把结果和第4个元素计算：f(9, 7)，结果为16；
再把结果和第5个元素计算：f(16, 9)，结果为25；
由于没有更多的元素了，计算结束，返回结果25。

三：
filter()函数是 Python 内置的另一个有用的高阶函数，filter()函数接收一个函数 f 和一个list，这个函数 f 的作用是对每个元素进行判断，返回 True或 False，filter()根据判断结果自动过滤掉不符合条件的元素，返回由符合条件元素组成的新list。
例如，要从一个list [1, 4, 6, 7, 9, 12, 17]中删除偶数，保留奇数，首先，要编写一个判断奇数的函数：

def is_odd(x):
    return x % 2 == 1

然后，利用filter()过滤掉偶数：

filter(is_odd, [1, 4, 6, 7, 9, 12, 17])

结果：[1, 7, 9, 17]

'''

import json
import requests
from requests import exceptions
URL = 'https://api.github.com'

def build_url(endpoint):
    return '/'.join([URL,endpoint])

def better_print(json_str):
    return json.dumps(json.loads(json_str),indent=4)

def request_method():
    response = requests.get(build_url('user/emails'),auth=('imoocdemo','imoocdemo123'))
    #print(response.text)
    print(better_print(response.text))

def params_request():
    response = requests.get(build_url('users'),params={'since':11})
    print(better_print(response.text))
    print(response.request.headers)
    print(response.url)

def json_request():
    response = requests.patch(build_url('user'),auth=('imoocdemo','imoocdemo123'),json={'name':'baby'})
    print(better_print(response.text))
    print(response.request.headers)
    print(response.request.body)
    print(response.status_code)

def timeout_request():
    try:
        response = requests.get(build_url('user/emails'),timeout=10)
        response.raise_for_status()
    except exceptions.Timeout as e:
        print(e)
    except exceptions.HTTPError as e:
        print(e)
    else:
        print(response.text)
        print(response.status_code)

def hard_requests():
    from requests import Request,Session
    s = Session()
    headers = {'User-Agent':'hys1.3.1'}
    req = Request('GET',build_url('user/emails'),auth=('imoocdemo','imoocdemo123'),headers=headers)
    prepped = req.prepare()
    print(prepped.body)
    print(prepped.headers)

    resp = s.send(prepped,timeout=10)
    print(resp.status_code)
    print(resp.request.headers)
    print(resp.text)

def error_request():
    response = requests.get('http://api.github.com')
    print(response.history)

def download_image():
    URL = 'https://timgsa.baidu.com/timg?image&quality=80&size=b9999_10000&sec=1508684596337&di=85cabd3af35d7b47e65de12719c7564a&imgtype=0&src=http%3A%2F%2Fpic.58pic.com%2F58pic%2F16%2F42%2F96%2F56e58PICAu9_1024.jpg'
    response = requests.get(URL,stream=True)
    print(response.status_code,response.reason)
    print(response.content)
    with open('demo.jpg','wb') as fd:
        for chunk in response.iter_content(128):
            fd.write(chunk)

def download_image_improved():
    URL = 'https://timgsa.baidu.com/timg?image&quality=80&size=b9999_10000&sec=1508684596337&di=85cabd3af35d7b47e65de12719c7564a&imgtype=0&src=http%3A%2F%2Fpic.58pic.com%2F58pic%2F16%2F42%2F96%2F56e58PICAu9_1024.jpg'
    response = requests.get(URL,stream=True)
    from contextlib import closing
    with closing(requests.get(URL,stream=True)) as response:
        with open('demo1.jpg','wb') as fd:
            for chunk in response.iter_content(128):
                fd.write(chunk)

if __name__ == '__main__':
    download_image_improved()


import datetime

import gym
import numpy as np
from gym import spaces, error
from gym import utils
from gym.utils import seeding
from entity.time_window import time_window
from tool.kuhn_munkras import kuhn_munkras

class MatchEnv(gym.Env):
    def __init__(self, max_car_num=30, max_lp_num=300, time_district_num=12, max_time_window_length=10):
        self.max_time_window_length = max_time_window_length
        self.action_spaces = spaces.Discrete(self.max_time_window_length)
        self.observation_space = spaces.MultiDiscrete(
            [max_car_num, max_lp_num, max_time_window_length, time_district_num])
        self.time_window = time_window
        self.matcher = kuhn_munkras(self.time_window)

    """
            #Description: 时间窗口内左右节点匹配
        """

    def match(self):
        self.matcher.change_batch(self.time_window)
        reward, match_res = self.matcher.km()
        unbound_lp_list = self.time_window.node_clear(match_res)
        self.time_window.drop_sent_load_plan(unbound_lp_list)
        return reward

    def time_minus(self, front_time_str: str, rear_time_str:str)->datetime.timedelta:
        front = datetime.datetime.strptime(front_time_str, "%Y%m%d%H%M%S")
        rear = datetime.datetime.strptime(rear_time_str, "%Y%m%d%H%M%S")
        tmp = rear - front
        return tmp

    @property
    def _get_obs(self):
        ob = np.zeros(4,dtype=int)
        time_str = self.time_window.time
        ob[0] = len(time_window.car_list)
        ob[1] = len(time_window.can_be_sent_load_plan)
        ob[2] = int(self.time_window.time[8:10]) / 2
        if len(self.time_window.car_list) != 0:
            first_car_time_Str = self.time_window.car_list[0].arrive_time
            time_delta = self.time_minus(first_car_time_Str, time_str)
            ob[3] = int(time_delta.seconds / 60) + 1
        else:
            ob[3] = 0
        return ob

    def step(self, a):
        reward = 0.0
        self.time_window.get_next_min()
        ob = self._get_obs
        if a <= ob[2] and a != 0:
            reward = self.match()
            ob = self._get_obs
        done = False
        if time_window.time >= time_window.end_time:
            done = True
        return ob, reward, done, {}

    def reset(self):
        self.time_window.reset()
        return self._get_obs


import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output

import pandas as pd
import plotly.express as px

df = pd.read_csv("../datasets/Los_Angeles_International_Airport_-_Passenger_Traffic_By_Terminal.csv")
df['ReportPeriod']=pd.to_datetime(df['ReportPeriod'])

external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']

app = dash.Dash(__name__, external_stylesheets=external_stylesheets)

app.layout = html.Div(children=[
    html.H1(children='ECS 272 Assignment 3'),

    html.Div(children='''
        Choose a terminal you are interested in.
    '''),

    dcc.Dropdown(
        id = 'terminal_dropdown',
        options = [
            {'label': 'Imperial Terminal', 'value': 'Imperial Terminal'},
            {'label': 'Misc. Terminal', 'value': 'Misc. Terminal'},
            {'label': 'Terminal 1', 'value': 'Terminal 1'},
            {'label': 'Terminal 2', 'value': 'Terminal 2'},
            {'label': 'Terminal 3', 'value': 'Terminal 3'},
            {'label': 'Terminal 4', 'value': 'Terminal 4'},
            {'label': 'Terminal 5', 'value': 'Terminal 5'},
            {'label': 'Terminal 6', 'value': 'Terminal 6'},
            {'label': 'Terminal 7', 'value': 'Terminal 7'},
            {'label': 'Terminal 8', 'value': 'Terminal 8'},
            {'label': 'Tom Bradley International Terminal', 'value': 'Tom Bradley International Terminal'},
        ],
        value = 'Imperial Terminal'
    ),

    dcc.Graph(
        id='basic_graph'
    ),

    html.Div(children='''
        Choose a year you are interested in.
    '''),

    dcc.Slider(
        id = 'year_slider',
        min = 2006,
        max = 2019,
        marks = {i: '{}'.format(i) for i in range(2006,2020)},
        value = 2006
    ),

    dcc.Graph(
        id='adv_graph'
    )
])

@app.callback(
    Output('basic_graph', 'figure'),
    [Input('terminal_dropdown', 'value')])
def update_basic_figure(terminal):
    figure={
        'data': [
            {'x': ["Domestic", "International"], 'y': [sum(df[df.Domestic_International == "Domestic"][df.Terminal == str(terminal)]["Passenger_Count"]),
                                                       sum(df[df.Domestic_International == "International"][df.Terminal == str(terminal)]["Passenger_Count"])],
                                                       'type': 'bar',
                                                       'name': 'temp'},
        ],
        'layout': {
            'title': 'Basic graph view'
        }
    }
    return figure

@app.callback(
    Output('adv_graph', 'figure'),
    [Input('year_slider', 'value')])
def update_advanced_figure(year):
    filtered_df = df[df['ReportPeriod'].dt.year == year][['Terminal', 'Domestic_International', 'Arrival_Departure']]
    figure = px.parallel_categories(filtered_df, dimensions_max_cardinality = 12)
    return figure

if __name__ == '__main__':
    app.run_server(debug=True)


import sys
import os
import imghdr
from PyQt5.QtWidgets import QApplication, QMainWindow, QAction
from PyQt5.QtWidgets import QWidget, QDesktopWidget, QMessageBox
from PyQt5.QtWidgets import QHBoxLayout, QVBoxLayout, QGridLayout
from PyQt5.QtWidgets import QGroupBox, QPushButton, QSlider
from PyQt5.QtWidgets import QLabel, QFileDialog, QSizePolicy, QSpacerItem
from PyQt5.QtGui import QIcon, QDrag, QPixmap, QImage
from PyQt5.QtCore import Qt, pyqtSignal
from timelapse_processing import ImageList, Image, loadImage, toRGB

"""
QApplication: manages application object.
QWidget: base class of all user interface objects. Receives events from the window system.
QMainWindow: main application window - framework to build the apps' user interface.
QDesktopWidget: provides access to user screen information.
"""

class DropButton(QPushButton):
    """
    Drag n Drop area widget
    """
    itemDropped = pyqtSignal(list)

    def __init__(self, title, width, height, parent):
        super().__init__(title, parent)
        self.setVisible(True)
        self.setAcceptDrops(True)
        self.setStyleSheet("background-color: rgba(85, 153, 255, 10%);"
                           "border-style: dashed;"
                           "border-width: 1px;"
                           "border-color: gray;"
                           "color: gray;")
        self.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding)
        self.setMinimumSize(width, height)

    def dragEnterEvent(self, e):
        m = e.mimeData()
        if m.hasUrls():
            e.accept()
        else:
            e.ignore()

    def dropEvent(self, e):
        m = e.mimeData()
        if m.hasUrls():
            e.accept()
            links = []
            [links.append(u.toLocalFile()) for u in m.urls()]
            self.itemDropped.emit(links)
            self.setVisible(False)
        else:
            e.ignore()

class TimelapseApp(QMainWindow):
    """
    Timelapse exposure fix application window
    """
    def __init__(self):
        super().__init__()
        self.title = 'Timelapse Exposure Fix'
        self.icon = 'camera_icon.png'
        self.left = 100
        self.top = 100
        self.width = 640
        self.height = 480
        self.minSize = 100
        self.imgScale = 0.45
        self.origImages = ImageList()
        self.processedImages = ImageList()
        self.imageFormat = ''
        self.initUI()

    def initUI(self):
        # Menu bar
        mainMenu = self.menuBar()
        self.fileMenu = mainMenu.addMenu('File')
        reloadAct = QAction('New Session', self)
        reloadAct.setShortcut('Ctrl+R')
        reloadAct.triggered.connect(self.reloadSession)
        reloadAct.setStatusTip('Reload a new session for timelapse processing')
        self.fileMenu.addAction(reloadAct)
        saveAct = QAction('Save Images', self)
        saveAct.setShortcut('Ctrl+S')
        saveAct.triggered.connect(self.saveImages)
        saveAct.setStatusTip('Save processed images')
        self.fileMenu.addAction(saveAct)
        saveAct.setDisabled(True)
        exitAct = QAction('Exit', self)
        exitAct.setShortcut('Ctrl+Q')
        exitAct.triggered.connect(self.close)
        exitAct.setStatusTip('Exit timelapse processing tool')
        self.fileMenu.addAction(exitAct)
        helpMenu = mainMenu.addMenu('Help')
        aboutAct = QAction('Drag n Drop', self)
        aboutAct.triggered.connect(self.helpWindow)
        aboutAct.setStatusTip('Drag n Drop information')
        helpMenu.addAction(aboutAct)

        # Grid layout
        self.createGridLayout()
        self.dragndrop.itemDropped.connect(self.pictureDropped)

        # Status bar
        self.statusBar()
        self.statusBar().setStyleSheet("background-color: white;")

        # Window settings
        self.setWindowTitle(self.title)
        self.setGeometry(self.left, self.top, self.width, self.height)
        self.setWindowIcon(QIcon(self.icon))
        self.center()
        self.show()

    def createGridLayout(self):
        self.centralWidget = QWidget(self)
        self.centralWidget.setStyleSheet("background-color: white;")
        self.centralWidget.setStatusTip('Ready')
        self.setCentralWidget(self.centralWidget)

        self.sld = QSlider(Qt.Horizontal, self)

        mainLayout = QHBoxLayout()
        vLayout = QVBoxLayout()
        grid = QGridLayout()
        grid.setSpacing(25)
        grid.setContentsMargins(25, 25, 25, 25)

        self.dragndrop = DropButton('Drop images here', self.width - self.minSize, self.minSize, self)
        grid.addWidget(self.dragndrop, 0, 0, 1, 4)

        self.viewerGroupBox = QGroupBox('Time-lapse viewer')
        self.img1 = QLabel(self.centralWidget)
        self.img1.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding)
        self.img1.setMinimumSize(self.minSize, self.minSize)

        self.img2 = QLabel(self.centralWidget)
        self.img2.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding)
        self.img2.setMinimumSize(self.minSize, self.minSize)

        imgviewer = QHBoxLayout()
        leftSpacer = QSpacerItem(self.minSize/10, self.minSize, QSizePolicy.Expanding, QSizePolicy.Minimum)
        imgviewer.addSpacerItem(leftSpacer)
        imgviewer.addWidget(self.img1)
        imgviewer.addWidget(self.img2)
        rightSpacer = QSpacerItem(self.minSize/10, self.minSize, QSizePolicy.Expanding, QSizePolicy.Minimum)
        imgviewer.addSpacerItem(rightSpacer)
        self.viewerGroupBox.setLayout(imgviewer)
        self.viewerGroupBox.setVisible(False)

        grid.addWidget(self.viewerGroupBox, 2, 1)

        grid.addWidget(self.sld, 3, 1)
        self.sld.setValue(0)
        self.sld.setRange(0,0)
        self.sld.valueChanged.connect(self.updateViewerIndex)
        self.sld.setVisible(False)

        # Encapsulate grid layout in VBox and HBox
        vLayout.addLayout(grid)
        verticalSpacer = QSpacerItem(self.minSize, 20, QSizePolicy.Minimum, QSizePolicy.Expanding)
        vLayout.addItem(verticalSpacer)
        hLeftSpacer = QSpacerItem(self.minSize/10, self.minSize, QSizePolicy.Expanding, QSizePolicy.Minimum)
        mainLayout.addItem(hLeftSpacer)
        mainLayout.addLayout(vLayout)
        hRightSpacer = QSpacerItem(self.minSize/10, self.minSize, QSizePolicy.Expanding, QSizePolicy.Minimum)
        mainLayout.addItem(hRightSpacer)
        self.centralWidget.setLayout(mainLayout)

    def updateViewerIndex(self):
        self.updateViewer(self.sld.value())

    def updateViewer(self, imageNumber):
        if len(self.origImages) > 0:
            rgb = toRGB(self.origImages[imageNumber].img)
            qimage = QImage(rgb, rgb.shape[1], rgb.shape[0], QImage.Format_RGB888)
            pixmap1 = QPixmap(qimage)
            pixmap1 = pixmap1.scaledToWidth(self.width * self.imgScale)
            self.img1.setPixmap(pixmap1)

        if len(self.processedImages) > 0:
            rgb = toRGB(self.processedImages[imageNumber].img)
            qimage = QImage(rgb, rgb.shape[1], rgb.shape[0], QImage.Format_RGB888)
            pixmap2 = QPixmap(qimage)
            pixmap2 = pixmap2.scaledToWidth(self.width * self.imgScale)
            self.img2.setPixmap(pixmap2)

    def pictureDropped(self, links):
        self.statusBar().showMessage('Processing...')
        self.imageFormat = imghdr.what(links[0])
        [self.origImages.append(Image(loadImage(link))) for link in links]
        newImages = ImageList(self.origImages[:])
        newImages.computeStats()
        newImages.fixExposure()
        self.processedImages = newImages

        self.fileMenu.actions()[1].setDisabled(False)

        self.updateViewer(0)
        self.sld.setRange(0, len(self.origImages) - 1)
        self.sld.setValue(0)
        self.statusBar().showMessage('Ready')

        self.viewerGroupBox.setVisible(True)
        self.sld.setVisible(True)

    def saveImages(self):
        self.statusBar().showMessage('Saving Images...')
        newDir = '/processed-images'
        destDir = QFileDialog.getExistingDirectory(self, "Select Directory") + newDir
        if not os.path.exists(destDir):
            os.makedirs(destDir)
        for i,obj in enumerate(self.processedImages):
            rgb = toRGB(obj.img)
            qimage = QImage(rgb, rgb.shape[1], rgb.shape[0], QImage.Format_RGB888)
            qimage.save(destDir + '/processed_image' + str(i+1).zfill(4) + '.' + self.imageFormat)
        self.statusBar().showMessage('Ready')

    def reloadSession(self):
        mboxtitle = 'Warning'
        mboxmsg = 'Are you sure you want to start a new session?\nAll unsaved changes will be lost.'
        reply = QMessageBox.warning(self, mboxtitle, mboxmsg,
                                    QMessageBox.Yes | QMessageBox.No, QMessageBox.No)
        if reply == QMessageBox.Yes:
            del self.origImages[:]
            del self.processedImages[:]
            self.img1.clear()
            self.img2.clear()
            self.sld.setValue(0)
            self.sld.setRange(0,0)
            self.dragndrop.setVisible(True)
            self.viewerGroupBox.setVisible(False)
            self.sld.setVisible(False)

    def helpWindow(self):
        mboxtitle = 'Help'
        mboxmsg = ('If your time-lapse is not displaying in the proper order, or there seems to be a jump '
        'cut to previous frames, make sure the last item clicked when the images were dragged was '
        'the first image of your sequence. The order of selection is preserved during drag and drop.')

        reply = QMessageBox.information(self, mboxtitle, mboxmsg,
                                        QMessageBox.Ok, QMessageBox.Ok)

    def center(self):
        qtRectangle = self.frameGeometry()
        centerPoint = QDesktopWidget().availableGeometry().center()
        qtRectangle.moveCenter(centerPoint)
        self.move(qtRectangle.topLeft())

    def closeEvent(self, event):
        mboxtitle = 'Message'
        mboxmsg = 'Are you sure you want to quit?'
        reply = QMessageBox.warning(self, mboxtitle, mboxmsg,
                                    QMessageBox.Yes | QMessageBox.No, QMessageBox.No)
        if reply == QMessageBox.Yes:
            event.accept()
        else:
            event.ignore()

if __name__ == '__main__':
    app = QApplication(sys.argv)
    w = TimelapseApp()
    sys.exit(app.exec_())


import matplotlib.patches as patches
import matplotlib.pyplot as plt

path = [
    [.1, .3],
    [.2, .9],
    [.8, .4],
]
fig = plt.figure()
ax = fig.gca()
ax.add_patch(patches.Polygon(path))
fig.savefig("triangle_patch.png", dpi=150)
plt.close()

path = [
    [.1, .3],
    [.2, .9],
    [.8, .4],
]
fig = plt.figure()
ax = fig.gca()
ax.add_patch(patches.Polygon(
  path,
  alpha=.7,
  edgecolor="darkblue",
  facecolor="red",
  hatch="+",
  joinstyle="miter",
  linestyle="--",
  linewidth=5,
))
fig.savefig("fancy_patch.png", dpi=150)
plt.close()


#!/usr/bin/env python3
#python crawl.py --iocp http://www.website.com/
import asyncio
import logging
import re
import signal
import os
import sys
import urllib.parse

import aiohttp

@asyncio.coroutine
def download(url,data):
    filename=url.replace('http://','').replace('https://','')
    path=filename.split('/')
    path='/'.join(path[0:-1])
    if not os.path.exists(path):os.makedirs(path)
    if os.path.exists(filename) or os.path.exists(filename+".html"):return
    if not filename.endswith('.html'):filename += 'index.html'
    with open(filename,'w') as f:
        f.write(data)

class Crawler:

    def __init__(self, rooturl, loop, maxtasks=100):
        self.rooturl = rooturl
        self.loop = loop
        self.todo = set()
        self.busy = set()
        self.done = {}
        self.tasks = set()
        self.sem = asyncio.Semaphore(maxtasks, loop=loop)

        # connector stores cookies between requests and uses connection pool
        self.session = aiohttp.ClientSession(loop=loop)

    @asyncio.coroutine
    def run(self):
        t = asyncio.ensure_future(self.addurls([(self.rooturl, '')]),
                                  loop=self.loop)
        yield from asyncio.sleep(1, loop=self.loop)
        while self.busy:
            yield from asyncio.sleep(1, loop=self.loop)

        yield from t
        yield from self.session.close()
        self.loop.stop()

    @asyncio.coroutine
    def addurls(self, urls):
        # print(urls)
        for url, parenturl in urls:
            url = urllib.parse.urljoin(parenturl, url)
            url, _ = urllib.parse.urldefrag(url)
            if (
                # url.startswith(self.rooturl) and
                    url not in self.busy and
                    url not in self.done and
                    url not in self.todo):
                self.todo.add(url)
                yield from self.sem.acquire()
                task = asyncio.ensure_future(self.process(url), loop=self.loop)
                task.add_done_callback(lambda t: self.sem.release())
                task.add_done_callback(self.tasks.remove)
                self.tasks.add(task)

    @asyncio.coroutine
    def process(self, url):
        print('processing:', url)

        self.todo.remove(url)
        self.busy.add(url)
        try:
            resp = yield from self.session.get(url)
        except Exception as exc:
            print('...', url, 'has error', repr(str(exc)))
            self.done[url] = False
        else:
            if (resp.status == 200 and
                    ('text/html' in resp.headers.get('content-type'))):
                data = (yield from resp.read()).decode('utf-8', 'replace')
                # print(url)
                yield download(url,data)
                urls = re.findall(r'(?i)href=["\']?([^\s"\'<>]+)', data)
                # print(urls)

                asyncio.Task(self.addurls([(u, url) for u in urls]))

            resp.close()
            self.done[url] = True

        self.busy.remove(url)
        print(len(self.done), 'completed tasks,', len(self.tasks),
              'still pending, todo', len(self.todo))

def main():
    loop = asyncio.get_event_loop()

    c = Crawler(sys.argv[1], loop)
    asyncio.ensure_future(c.run(), loop=loop)

    try:
        loop.add_signal_handler(signal.SIGINT, loop.stop)
    except RuntimeError:
        pass
    loop.run_forever()
    print('todo:', len(c.todo))
    print('busy:', len(c.busy))
    print('done:', len(c.done), '; ok:', sum(c.done.values()))
    print('tasks:', len(c.tasks))

if __name__ == '__main__':
    if '--iocp' in sys.argv:
        from asyncio import events, windows_events
        sys.argv.remove('--iocp')
        logging.info('using iocp')
        el = windows_events.ProactorEventLoop()
        events.set_event_loop(el)

    main()


import keras
from keras.models import Sequential
from keras.layers import Dense
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold

#设置随机数种子
np.random.seed(7)

#数据路径
data_file="pima-indians-diabetes.csv"

#导入数据
dataset=np.loadtxt(data_file,encoding='utf-8',delimiter=',')
#分割X和y
X=dataset[:,0:8]
y=dataset[:,8]

#10折,训练集和验证集总数共为10
kfold=StratifiedKFold(n_splits=10,shuffle=True,random_state=7)

cvscores=[]

for train,validation in kfold.split(X,y):
    # 创建模型
    model = Sequential()
    model.add(Dense(12, input_dim=8, activation='relu'))
    model.add(Dense(6, activation='relu'))

    # 最后一层dense，sigmoid输出患病概率
    model.add(Dense(1, activation='sigmoid'))

    # 编译模型
    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])

    # 训练模型
    model.fit(X, y, epochs=150, batch_size=10, verbose=0)

    # 评估模型
    scores = model.evaluate(X[validation], y[validation],verbose=0)

    #输出评估结果
    print('\n%s : %.2f%%' % (model.metrics_names[1], scores[1] * 100))

    cvscores.append(scores[1]*100)

#输出均值和标准差
print("%.2f%% (+/- %.2f%%)" % (np.mean(cvscores),np.std(cvscores)))


from flask import jsonify, make_response, Blueprint, abort, redirect, request
from atexit import register
from time import sleep
from threading import Thread
from flasgger import swag_from
import utils
import logging
import json

bp = Blueprint('db', __name__)
search_q = utils.RedisQueue('searched')
expand_q = utils.RedisQueue('expanded')

def on_startup():
    def acquire_user(q, threshold):
        while True:
            try:
                if q.qsize() < threshold:
                    rows = db.view(f'tree/{q.name}', limit=threshold, reduce=False,
                                   include_docs=True)[[False]:[False, {}]].rows
                    users = []
                    for row in rows:
                        user = row.doc
                        user[q.name] = 'queue'
                        users.append(user)

                    if users:   # wait for 5 seconds to next fetch
                        results = db.update(users)
                        logging.warning(
                            f'Put {sum([r[0] for r in results])} into {q.name} queue ...')
                        for user, (suc, uid, rev) in zip(users, results):
                            if suc:
                                user['_rev'] = rev
                                q.put(user)
                    else:
                        sleep(5)
                else:
                    sleep(1)
            except Exception as e:
                logging.error(repr(e))
                sleep(1)

    Thread(target=acquire_user, daemon=True, args=(search_q, 200,)).start()
    Thread(target=acquire_user, daemon=True, args=(expand_q, 10,)).start()

@register
def on_exit():
    for q in [search_q, expand_q]:
        users = []
        while not q.empty():
            user = json.loads(q.get())
            user['searched'] = False
            users.append(user)
        result = db.update(users)
        logging.warning(
            f'Put back {sum([r[0] for r in result])} users from {q.name} queue to db.')

db = utils.db(name='users')
on_startup()

@bp.route('/monitor/<level>/')
@swag_from('docs/db_monitor.yml')
def monitor(level):
    search = []
    for item in db.view('tree/searched', group_level=level):
        search.append({str(item.key): item.value})
    return {
        '# of Docs': utils.db().info()['doc_count'],
        'Search': search
    }

@bp.route('/next_search')
@swag_from('docs/next_search.yml')
def next_search():
    user = search_q.get_nowait()
    return user if user else {}

@bp.route('/next_expand')
@swag_from('docs/next_expand.yml')
def next_expand():
    user = expand_q.get_nowait()
    return user if user else {}

@bp.route('/queue_stat')
@swag_from('docs/q_stat.yml')
def q_stat():
    return {'search': search_q.qsize(), 'expand': expand_q.qsize()}

@bp.route('/active_tasks')
def active_tasks():
    return redirect(f'{utils.base()}/_active_tasks')


# -*- coding: utf-8 -*-

""" Autoruns
    2015 fightnight
    2022 bittor7x0"""

import xbmc,xbmcvfs,xbmcaddon,xbmcgui,xbmcplugin,urllib.request,urllib.parse,urllib.error,os,re,sys
import xml.etree.ElementTree as ET

SERVICE_DISABLED = 'Autoruns_service_disabled'

def list_addons():
      #info directory
      addDir('[COLOR blue][B]%s[/B][/COLOR]' % (translate(30001)),'None',None,xbmcaddon.Addon().getAddonInfo('icon'))

      #get the path of addons
      pathofaddons = xbmcvfs.translatePath('special://home/addons')

      #list with addons
      listofaddons = os.listdir(pathofaddons)
      for individual_addon in listofaddons:
            #path to individual addon, cycle for all the addons
            path_to_addon = os.path.join(pathofaddons, individual_addon)

            #define addon.xml path
            addon_xml_path=os.path.join(path_to_addon,'addon.xml')

            #check the existence of addon.xml, if true, we continue
            if os.path.exists(addon_xml_path):

                  #get addon.xml content
                  xml_content=openfile(addon_xml_path)

                  #get icon from addon.xml because creating a xbmcaddon.Addon(individual_addon) object causes the exception "Unknown addon id 'xxxx'" with installed addons but not enabled
                  root = ET.fromstring(xml_content)
                  xml_icon = root.findall("./extension/assets/icon")
                  addon_icon_path = os.path.join(path_to_addon,xml_icon[0].text if xml_icon else 'icon.png')

                  if re.search(SERVICE_DISABLED,xml_content):
                        #addon with service off
                        addDir('[B][COLOR gold]%s[/B] (off)[/COLOR]' % (individual_addon),path_to_addon,1,addon_icon_path)
                  elif re.search('point="xbmc.service"',xml_content):
                        #addon with service on
                        addDir('%s (on)' % (individual_addon),path_to_addon,1,addon_icon_path)
                  else:
                        #addon with no service
                        pass

def change_state(name,path):
      #define addon.xml path to change
      addon_xml_path=os.path.join(path,'addon.xml')

      #get addon.xml content
      content=openfile(addon_xml_path)

      if re.search('COLOR gold',name):
            #service off to on, so we uncomment the service element
            content=content.replace('<!--%s ' % (SERVICE_DISABLED),'').replace(' %s-->' % (SERVICE_DISABLED),'')
      else:
            #service on to off, so we comment the service element
            root = ET.fromstring(content)
            parent = root.findall("./extension[@point='xbmc.service']/..")[0]
            child = parent.findall("./extension[@point='xbmc.service']")[0]
            commented = '%s \n%s %s' % (SERVICE_DISABLED, ET.tostring(child).decode("utf-8"), SERVICE_DISABLED)
            parent.remove(child)
            parent.append(ET.Comment(commented))
            content=ET.tostring(root, encoding='utf8', method='xml')

      #change state on addon.xml
      savefile(addon_xml_path,content)

      #refresh the list
      xbmc.executebuiltin("Container.Refresh")

def openfile(path_to_the_file):
    try:
        fh = xbmcvfs.File(path_to_the_file, 'rb')
        contents=fh.read()
        fh.close()
        return contents
    except:
        print("Wont open: %s" % filename)
        return None

def savefile(path_to_the_file,content):
    try:
        fh = xbmcvfs.File(path_to_the_file, 'wb')
        fh.write(content)
        fh.close()
    except: print("Wont save: %s" % filename)

def addDir(name,path,mode,iconimage):
      listItem = xbmcgui.ListItem(label=name)
      listItem.setArt({'icon': "DefaultFolder.png"})
      listItem.setArt({'thumb': iconimage})
      return xbmcplugin.addDirectoryItem(handle=int(sys.argv[1]),url="%s?path=%s&mode=%s&name=%s" % (sys.argv[0],urllib.parse.quote_plus(path),mode,urllib.parse.quote_plus(name)),listitem=listItem,isFolder=False)

def get_params():
      param=[]
      paramstring=sys.argv[2]
      if len(paramstring)>=2:
            params=sys.argv[2]
            cleanedparams=params.replace('?','')
            if (params[len(params)-1]=='/'):
                  params=params[0:len(params)-2]
            pairsofparams=cleanedparams.split('&')
            param={}
            for i in range(len(pairsofparams)):
                  splitparams={}
                  splitparams=pairsofparams[i].split('=')
                  if (len(splitparams))==2:
                        param[splitparams[0]]=splitparams[1]
      return param

def translate(text):
      return xbmcaddon.Addon().getLocalizedString(text)

params=get_params()
path=None
name=None
mode=None

try: path=urllib.parse.unquote_plus(params["path"])
except: pass
try: name=urllib.parse.unquote_plus(params["name"])
except: pass
try: mode=int(params["mode"])
except: pass

if mode==None: list_addons()
elif mode==1: change_state(name,path)

xbmcplugin.endOfDirectory(int(sys.argv[1]))


import threading
import numpy as np
import cv2
import time

class CameraReader(threading.Thread):
    def __init__(self,dev):
        self.dev = dev
        self.lock = threading.Lock()
        self.frame = np.empty((480,640,3), dtype=np.uint8)
        self.running = False
        self.cap = None
        threading.Thread.__init__(self)

    def run(self):
        while self.running:
            self.lock.acquire()
            ret, _ = self.cap.read(self.frame)
            self.lock.release()
            time.sleep(0.01)

    def __enter__(self):
        self.cap = cv2.VideoCapture(self.dev)
        self.running = True
        self.start()
        return self

    def __exit__(self,exception_type, exception_value, traceback):
        self.stop()
        if self.cap is not None:
            self.cap.release()

    def read(self,frame=None):
        self.lock.acquire()
        if frame is None:
            frame = self.frame.copy()
        else:
            np.copyto(frame,self.frame)
        self.lock.release()
        return frame

    def stop(self):
        self.running = False

import rospy
from cv_bridge import CvBridge
from sensor_msgs.msg import Image
class ROSCameraReader(object):
    def __init__(self, topic):
        self.topic = topic
        self.sub = None
        self.bridge = CvBridge()
        self.img = np.zeros((480,640,3), dtype=np.uint8)
    def __enter__(self):
        self.sub = rospy.Subscriber(self.topic, Image, self.img_cb)
        return self
    def __exit__(self,a,b,c):
        if self.sub is not None:
            self.sub.unregister()
            self.sub = None
    def img_cb(self,msg):
        self.img = self.bridge.imgmsg_to_cv2(msg, desired_encoding="bgr8")
    def read(self,frame):
        np.copyto(frame, self.img)


from pymongo import MongoClient
import numpy as np
import datetime

client = MongoClient()
db = client.twitter

cursor = db.tweets.aggregate(
    [{
        "$project": {
            "y": { "$year": "$timestamp_obj"},
            "m": { "$month": "$timestamp_obj"},
            "d": { "$dayOfMonth": "$timestamp_obj" },
            "h": { "$hour": "$timestamp_obj" }
        }
    },
    {
        "$group": {
            "_id": { "year": "$y", "month": "$m", "day": "$d", "hour": "$h" },
            "count": { "$sum": 1 }
        }
    },
    {
        "$sort": {
            "_id.year": -1,
            "_id.month": 1,
            "_id.day": 1,
            "_id.hour": 1
        }
    }]
)

hours = [[] for i in range(24)]
counts = {}
earliest, latest = None, None

for doc in cursor:
    d = doc['_id']
    time = datetime.datetime(int(d['year']), int(d['month']), int(d['day']), int(d['hour']))

    counts[time] = doc['count']

    if earliest == None or time < earliest: earliest = time
    if latest == None or time > latest: latest = time

    hours[int(d['hour'])].append(int(doc['count']))
    print (str(doc).encode('utf-8'))

for i in range(0, len(hours)):
    hours[i] = sum(hours[i]) / len(hours[i])

starttime = earliest
sortedcounts = []
sortedhours = []
while not starttime >= latest:
    try:
        sortedcounts.append(counts[starttime])
        sortedhours.append(hours[starttime.hour])
    except:
        sortedcounts.append(0)
        sortedhours.append(0)

    starttime += datetime.timedelta(hours=1)

N = len(sortedcounts)
# sample spacing
T = 1.0 / 100.0

yf = np.fft.fft(sortedcounts) # fft computing and normalization
xf = np.linspace(0.0, 1.0/(2.0*T), N/2)

yf[0] = 0

import matplotlib.pyplot as plt

fig, ax = plt.subplots(4, 1)

ax[0].plot(sortedcounts)
ax[0].set_xlabel('Time')
ax[0].set_ylabel('Count')

ax[1].plot(xf, 2.0/N * np.abs(yf[:N/2])) # plotting the spectrum

print(yf)
for i in range(0, 27):
    yf[i] = 0

for i in range(28, len(yf)-1):
    yf[i] = 0

ax[2].plot(xf, 2.0/N * np.abs(yf[:N/2]), 'rx') # plotting the spectrum

ixf = np.fft.ifft(yf)
ax[3].plot(ixf)
ax[3].plot(sortedcounts)
ax[3].plot(sortedhours)
ax[3].set_xlabel('Time')
ax[3].set_ylabel('Count')

plt.show()


# -*- coding: utf-8 -*-

from ..enums import ScanTypesEnum
from ..errors import ClosedProcess
from ..process import AbstractProcess
from .functions import (
    get_memory_regions,
    read_process_memory,
    search_addresses_by_value,
    search_values_by_addresses,
    write_process_memory
)
from typing import Generator, Optional, Sequence, Tuple, Type, TypeVar, Union

T = TypeVar("T")

class LinuxProcess(AbstractProcess):
    """
    Class to open a Linux process for reading and writing memory.
    """

    def __init__(
        self,
        *,
        window_title: Optional[str] = None,
        process_name: Optional[str] = None,
        pid: Optional[int] = None,
        **kwargs
    ):
        """
        :param window_title: window title of the target program.
        :param process_name: name of the target process.
        :param pid: process ID.
        """
        super().__init__(
            window_title = window_title,
            process_name = process_name,
            pid = pid
        )
        self.__closed = False

    def close(self) -> bool:
        """
        Close the process handle.
        """
        self.__closed = True
        return True

    def get_memory_regions(self) -> Generator[dict, None, None]:
        """
        Generates dictionaries with the address, size and other
        information of each memory region used by the process.
        """
        if self.__closed: raise ClosedProcess()

        return get_memory_regions(self.pid)

    def read_process_memory(
        self,
        address: int,
        pytype: Type[T],
        bufflength: int
    ) -> T:
        """
        Return a value from a memory address.

        :param address: target memory address (ex: 0x006A9EC0).
        :param pytype: type of the value to be received (bool, int, float, str or bytes).
        :param bufflength: value size in bytes (1, 2, 4, 8).
        """
        if self.__closed: raise ClosedProcess()

        return read_process_memory(self.pid, address, pytype, bufflength)

    def search_by_addresses(
        self,
        pytype: Type[T],
        bufflength: int,
        addresses: Sequence[int],
        *,
        raise_error: bool = False,
    ) -> Generator[Tuple[int, Optional[T]], None, None]:
        """
        Search the whole memory space, accessible to the process,
        for the provided list of addresses, returning their values.
        """
        if self.__closed: raise ClosedProcess()
        return search_values_by_addresses(self.pid, pytype, bufflength, addresses, raise_error=raise_error)

    def search_by_value(
        self,
        pytype: Type[T],
        bufflength: int,
        value: Union[bool, int, float, str, bytes],
        scan_type: ScanTypesEnum = ScanTypesEnum.EXACT_VALUE,
        *,
        progress_information: bool = False,
        writeable_only: bool = False,
    ) -> Generator[Union[int, Tuple[int, dict]], None, None]:
        """
        Search the whole memory space, accessible to the process,
        for the provided value, returning the found addresses.

        :param pytype: type of value to be queried (bool, int, float, str or bytes).
        :param bufflength: value size in bytes (1, 2, 4, 8).
        :param value: value to be queried (bool, int, float, str or bytes).
        :param scan_type: the way to compare the values.
        :param progress_information: if True, a dictionary with the progress information will be return.
        :param writeable_only: if True, search only at writeable memory regions.
        """
        if self.__closed: raise ClosedProcess()

        if scan_type in [ScanTypesEnum.VALUE_BETWEEN, ScanTypesEnum.NOT_VALUE_BETWEEN]:
            raise ValueError("Use the method search_by_value_between(...) to search within a range of values.")

        return search_addresses_by_value(self.pid, pytype, bufflength, value, scan_type, progress_information, writeable_only)

    def search_by_value_between(
        self,
        pytype: Type[T],
        bufflength: int,
        start: Union[bool, int, float, str, bytes],
        end: Union[bool, int, float, str, bytes],
        *,
        not_between: bool = False,
        progress_information: bool = False,
        writeable_only: bool = False,
    ) -> Generator[Union[int, Tuple[int, dict]], None, None]:
        """
        Search the whole memory space, accessible to the process,
        for a value within the provided range, returning the found addresses.

        :param pytype: type of value to be queried (bool, int, float, str or bytes).
        :param bufflength: value size in bytes (1, 2, 4, 8).
        :param start: minimum inclusive value to be queried (bool, int, float, str or bytes).
        :param end: maximum inclusive value to be queried (bool, int, float, str or bytes).
        :param not_between: if True, return only addresses of values that are NOT within the range.
        :param progress_information: if True, a dictionary with the progress information will be return.
        :param writeable_only: if True, search only at writeable memory regions.
        """
        if self.__closed: raise ClosedProcess()

        scan_type = ScanTypesEnum.NOT_VALUE_BETWEEN if not_between else ScanTypesEnum.VALUE_BETWEEN
        return search_addresses_by_value(self.pid, pytype, bufflength, (start, end), scan_type, progress_information, writeable_only)

    def write_process_memory(
        self,
        address: int,
        pytype: Type[T],
        bufflength: int,
        value: Union[bool, int, float, str, bytes]
    ) -> T:
        """
        Write a value to a memory address.

        :param address: target memory address (ex: 0x006A9EC0).
        :param pytype: type of value to be written into memory (bool, int, float, str or bytes).
        :param bufflength: value size in bytes (1, 2, 4, 8).
        :param value: value to be written (bool, int, float, str or bytes).
        """
        if self.__closed: raise ClosedProcess()

        return write_process_memory(self.pid, address, pytype, bufflength, value)


import numpy as np
import matplotlib.pyplot as plt

# 1. Linear regression for classifying noisy data

fig, axs = plt.subplots(3)
N = 100     # data set size
d = 2       # 2 classes of data

# Generate random training data
X = np.random.uniform(-1, 1, size=(N, d+1))
X[:, 0] = 1
# Calculate weights vector
w = np.random.uniform(-1, 1, size=(d+1))
# Compute true labels for the training data
Y = np.sign(np.dot(X, w))

# Add Noise
n = np.random.randint(0, 100, 1)
Y[n[0]] = Y[n[0]]*-1

ind_pos = np.where(Y == 1)[0]   # positive examples
ind_neg = np.where(Y == -1)[0]  # negative examples

# Plot points
axs[0].plot(X[ind_pos, 1], X[ind_pos, 2], 'ro')    # red dot points
axs[0].plot(X[ind_neg, 1], X[ind_neg, 2], 'bx')    # blue 'x' points

# Generate random target function: f(x) = w^Tx
X2 = (-w[1]/w[2]*X) - w[0]/w[2]
line_x = np.linspace(-1, 1, 100)
# Plot target function
axs[0].plot(X, X2, label='Target fxn', color='yellow')

# Perceptron algorithm
def perceptron(X, Y, pocket=False):
    # List of in-sample errors
    eins = []
    best_ein = 1000
    # To be trained
    w_train = np.random.uniform(-1, 1, size=(d + 1))
    best_w = w_train

    for i in range(1000):
        Y1 = np.sign(np.dot(X, w_train))
        # Check for misclassified point
        j = np.random.randint(0, 100, 1)

        if Y1[j] != Y[j]:
            # Update hypothesis
            w_train += X[j][0]*Y[j]

        # Calculate error
        ein = np.count_nonzero(Y1-Y) / 100
        if pocket:
            if ein < best_ein:
                best_w = w_train
                best_ein = ein  ###
        eins.append(best_ein if pocket else ein)

    return best_w if pocket else w_train, eins

# Run PLA
l = np.linspace(0, 1000, 1000)
result = perceptron(X, Y, pocket=False)
g = result[0]
g_x = -(g[0]/g[2]) + (-(g[1]*X)/g[2])
ein_pla = result[1]
axs[0].plot(X, g_x, label='PLA final hypothesis', color='orange')
axs[1].plot(l, ein_pla, label='Ein PLA', color='green')

# Run Pocket Algorithm
result2 = perceptron(X, Y, pocket=True)
g2 = result2[0]
g_x2 = -(g2[0]/g2[2]) + (-(g2[1]*X)/g2[2])
ein_pocket = result2[1]
axs[0].plot(X, g_x2, label='Pocket final hypothesis', color='blue')
axs[2].plot(l, ein_pocket, label='Ein Pocket', color='red')

# Linear Regression     eqn 3.4
Xdag = np.matmul(np.linalg.pinv(np.matmul(X.transpose(), X)), X.transpose())
w = np.matmul(Xdag, Y)
wTx = np.matmul(w, X.transpose())
xw = np.matmul(X, w)
p1 = np.matmul(wTx, xw)
p2 = np.multiply(np.matmul(wTx, Y), 2)
p3 = np.matmul(Y.transpose(), Y)
ein_lin_reg = np.add(np.subtract(p1, p2), p3)/100
print(ein_lin_reg)

# Plot Linear Regression
lin_reg = (-(w[0])/w[2]) + (-w[1]*X/w[2])
axs[0].plot(X, lin_reg, label='Linear Regression', color='purple')

plt.show()

Linear Regression, PLA, and Pocket Algorithm/Lab3_Code.py


import socket
from time import sleep

# Keyence Scanner ASCII Commands ------------------------------------------------------

"""
Error Format: ER,COMMAND,ERROR_CODE

Error Codes:
------------
00  Undefined command received
01  Mismatched command format (Invalid number of parameters)
02  The parameter 1 value exceeds the set value
03  The parameter 2 value exceeds the set value
04  Parameter 2 is not set in HEX (hexadecimal) code
05  Parameter 2 set in HEX (hexadecimal) code but exceeds the set value
10  There are 2 or more ! marks in the preset data, Preset data is incorrect
11  Area specification data is incorrect
12  Specified file does not exist
13  "mm" for the %Tmm-LON,bb command exceeds the setting range.
14  Communication cannot be checked with the %Tmm-KEYENCE command.
20  This command is not executable in the current status (execution error)
21  The buffer has overflowed, so commands cannot be executed
22  An error occurred while loading or saving parameters, so commands cannot be executed
23  Commands sent from RS-232C cannot be received because AutoID Network Navigator is being connected.
99  SR-1000 Series may be faulty. Please contact your nearest KEYENCE sales office
"""

# Read Serial Number
TRIGGER_INPUT_ON = b'LON\r'
TRIGGER_INPUT_OFF = b'LOFF\r'

REQUEST_SCANNER_VERSION = b'KEYENCE\r'

# Response: OK,FTUNE
REQUEST_FOCUS_ADJUSTMENT = b'FTUNE\r'

# Set the timing mode to "One-shot trigger" (one-shot signal trigger)
SET_ONESHOT_TRIGGER = b'WP,101,1\r'
# Check if one shot trigger is set
IS_ONESHOT_TRIGGER = b'RP,101\r'

SET_TIME_APPENDING = b'WP,300,1\r'
IS_TIME_APPENDING = b'RP,300,1\r'

""" Response:
OK,NUM,a,b,c,d,e
    a: OK count
    b: NG count
    c: ERROR count
    d: STABLE count
    e: Trigger input count (0 to 65535)
"""
READING_HISTORY = b'NUM\r'

""" Response:
OK,ERRSTAT,m
    m = None:       No error
        system:     System error
        update:     Update error
        cfg:        Set value error
        ip:         IP address duplication
        over:       Buffer overflow
        plc:        PLC link error
        profinet:   PROFINET error
        lua:        Script error
        hostconnect: Host connection erro
"""
GET_ERROR_STATUS = b'ERRSTAT\r'

# -----------------------------------------------------------------------------

SOCKET_TIMEOUT = 10

"""
Class for Communication with an SR1000 Keyence Scanner
"""
class Scanner:

    def __init__(self, ip_addr, port, logger=None):
        self.ip = ip_addr
        self.port = port
        self.conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.conn.settimeout(SOCKET_TIMEOUT)
        self.connected = False
        self.log = logger
        self.connect()

    # Initiate a connection to the SR1000
    def connect(self):
        try:
            self.conn.connect((self.ip, self.port))  # connect to the server
            sleep(.1)
            # print(self.rx())
            self.connected = True
            print(f"Connected to {self.ip}:{self.port}")
        except socket.error as er:
            print(f"Connection Failed: {er}")

    def tx(self, message):
        self.conn.sendall(message)  # send message

    def rx(self):
        data = self.conn.recv(1024).decode()  # receive response
        # print('Received from scanner: ' + data)
        return data

    # Send/Receive a Message
    def req(self, message):
        self.tx(message)  # send TCP message
        sleep(.1)
        data = self.rx()  # recieve TCP message
        return data
        # self.conn.close()  # close the connection

    # Request Status
    def get_status(self):
        # return self.req(REQUEST_STATUS)
        pass

    # Send/Receive a Message
    def read_code(self):
        return str(self.req(TRIGGER_INPUT_ON))

    def error_type(self):
        return self.req(GET_ERROR_STATUS).replace('OK,ERRSTAT,', '')

if __name__ == '__main__':

    from dotenv import dotenv_values
    c = dotenv_values()

    s = Scanner(c["SR1000_SCANNER_IP"], int(c["SR1000_SCANNER_PORT"]))

    try:

        while True:
            # Fetch serial number
            print(s.read_code())
            sleep(1)

    except KeyboardInterrupt:
        print("Done")


import tweepy
import time

#creds
auth = tweepy.OAuthHandler('F65nYoxP5rSF2GCvhYmyGTgf9' , 'ur8QEgt7J2ugicpmSCLauprXMNOVOzIfAQMOAXnbhflvb2aH1W')
auth.set_access_token('1309177113652662272-4rrOUk9Pxn9KccGPnATO2IUypgT8tr', '1K3NjvrLHuMYeR4xfpp2CpLENsgqrPugZ4WvTtI6ZAyO5' )

api = tweepy.API(auth)
user = api.me()
#print("user creds: ", user)

#api.update_status("Hello, this is a test with bukola")

#for following in tweepy.Cursor(api.friends).items():
 #   print(following.name)

#randomly likes a tweet based on keywords
# search = 'UniversityofHouston'
# num_tweet = 2

# for tweet in tweepy.Cursor(api.search, search).items(num_tweet):
#     try:
#         print("tweet liked")
#         tweet.favorite()
#         time.sleep(1)

#     except tweepy.TweepyError as e:
#         print(e.reason)

#     except StopIteration:
#         break

tweet_account = api.user_timeline(screen_name = "jay_hopee")
for tweet in tweet_account:
    if tweet.text[0:2] != "RT":
        tweet.favorite()
        print("liked my own tweet")


from graph_tool.all import Graph,Vertex,graph_draw,radial_tree_layout
from GN0.alpha_zero.MCTS_cached import MCTS as MCTS_old,Node,Leafnode,upper_confidence_bound
from GN0.alpha_zero.MCTS import MCTS
from GN0.alpha_zero.NN_interface import NNetWrapper
from graph_game.shannon_node_switching_game import Node_switching_game
from graph_game.graph_tools_games import get_graph_only_hex_game,Hex_game
import numpy as np
from typing import Union
import os
import time
from GN0.models import get_pre_defined
from argparse import Namespace
from graph_game.graph_tools_hashing import get_unique_hash
import torch

def dummy_nn(game:Node_switching_game):
    moves = game.get_actions()
    prob = np.array(list(range(len(moves))),dtype=float)+1
    prob/=np.sum(prob)
    value = 0.7 if len(moves)%2==0 else 0.3
    return moves,torch.from_numpy(prob),torch.tensor([value])

def graph_state(game,prob=None):
    if prob is None:
        prior_map = None
    else:
        prior_map = game.view.new_vertex_property("double")
        for i,vertex in enumerate(game.get_actions()):
            prior_map[vertex] = prob[i]
    os.system("pkill -f 'mupdf node_state.pdf'")
    game.draw_me(fname="node_state.pdf",vprop1=prior_map,decimal_places=3)
    os.system("nohup mupdf node_state.pdf > /dev/null 2>&1 &")

def graph_node(node:Node,prob=None):
    if prob is None:
        prob = node.priors
    game = Node_switching_game.from_graph(node.storage)
    prior_map = game.view.new_vertex_property("double")
    for vertex in game.get_actions():
        prior_map[vertex] = prob[list(node.moves).index(vertex)]

    os.system("pkill -f 'mupdf node_state.pdf'")
    game.draw_me(fname="node_state.pdf",vprop1=prior_map,decimal_places=3)
    os.system("nohup mupdf node_state.pdf > /dev/null 2>&1 &")

def graph_from_hash_mcts(mcts:MCTS,root:Node_switching_game,to_show="num_visits",last_hash=int):
    def recursive_add(state:Graph,cur_vertex:Vertex):
        game.set_to_graph(Graph(state))
        s = get_unique_hash(game.view)
        if s not in mcts.Ps:
            return
        actions = game.get_actions()
        if s in mcts.Qsa:
            ucb_many = mcts.Qsa[s] + mcts.cpuct * mcts.Ps[s] * np.sqrt(mcts.Ns[s]) / (1 + mcts.Nsa[s])
        else:
            ucb_many = 0.5 + mcts.cpuct * mcts.Ps[s] * np.sqrt(mcts.Ns[s])
        nsa = mcts.Nsa[s] if s in mcts.Nsa else np.zeros_like(mcts.Ps[s])
        qsa = mcts.Qsa[s] if s in mcts.Qsa else np.ones_like(mcts.Ps[s])*0.5
        for ucb,prior,visits,q,action in zip(ucb_many,mcts.Ps[s],nsa,qsa,game.get_actions()):
            game.set_to_graph(Graph(state))
            game.make_move(action)
            new_state = Graph(game.graph)
            new_s = get_unique_hash(game.view)
            v = g.add_vertex()
            g.add_edge(cur_vertex,v)
            color[v] = "blue" if state.gp["m"] else "red"
            if to_show=="num_visits":
                text[v] = str(visits)
            elif to_show == "q":
                text[v] = f"{q:.2f}"[1:] if q<1 else "1"
            elif to_show == "prior":
                text[v] = f"{prior:.2f}"
            elif to_show == "m":
                text[v] = str(action)
            elif to_show=="numbers":
                text[v] = str(int(v))
            elif to_show=="ucb":
                text[v] = f"{float(ucb):.2f}"

            number_to_state[int(v)] = new_state
            if last_hash==new_s:
                halo_color[v] = [0,1,0,0.5]
                weather_halo[v] = True
            else:
                weather_halo[v] = False
                halo_color[v] = [0,0,0,0]
            if new_s in mcts.Ps:
                shape[v] = "circle"
                recursive_add(new_state,v)
            else:
                shape[v] = "square"

    state = Graph(root.graph)
    game = root

    number_to_state = {}
    g = Graph(directed=True)
    color = g.new_vertex_property("string")
    shape = g.new_vertex_property("string")
    text = g.new_vertex_property("string")
    size = g.new_vertex_property("int")
    pen_width = g.new_vertex_property("int")
    halo_color = g.new_vertex_property("vector<double>")
    weather_halo = g.new_vertex_property("bool")
    halo_size = g.new_vertex_property("double")

    g.vp.l = halo_size
    g.vp.h = halo_color
    g.vp.g = size
    g.vp.c = color
    g.vp.s = shape
    g.vp.t = text
    g.vp.p = pen_width
    g.vp.b = weather_halo

    root_hash = get_unique_hash(game.view)
    number_to_state[0] = Graph(state)

    v = g.add_vertex()
    halo_color[v] = [0,0,0,0]
    if root_hash==last_hash:
        halo_color[v] = [0,1,0,0.5]
        weather_halo[v] = True

    recursive_add(state,v)

    halo_size.a = 1.1
    makerturn = state.gp["m"]
    color[v] = "red" if makerturn else "blue"
    shape[v] = "circle" if root_hash in mcts.Ps else "square"
    text[v] = ""
    size.a = 25
    return g,number_to_state

def graph_from_root(root:Union[Node,Leafnode],to_show="num_visits",last_node=None):
    def recursive_add(cur_node:Node,cur_vertex:Vertex):
        ucb = upper_confidence_bound(cur_node,1)
        for tv,child,q,visits,one_ucb,prior in zip(cur_node.total_value,cur_node.children,cur_node.Q,cur_node.visits,ucb,cur_node.priors):
            v = g.add_vertex()
            g.add_edge(cur_vertex,v)
            color[v] = "blue" if cur_node.storage.gp["m"] else "red"
            if to_show=="num_visits":
                text[v] = str(visits)
            elif to_show == "q":
                text[v] = f"{q:.2f}"[1:] if q<1 else "1"
            elif to_show == "value":
                if isinstance(child,Node):
                    text[v] = f"{tv:.2f}"
                else:
                    text[v] = child.value
            elif to_show == "prior":
                text[v] = f"{prior:.2f}"
            elif to_show == "m":
                if isinstance(child,Node):
                    text[v] = ""
                else:
                    text[v] = "" if child.move is None else child.move
            elif to_show=="numbers":
                text[v] = str(int(v))
            elif to_show=="ucb":
                text[v] = f"{float(one_ucb):.2f}"

            number_to_node[int(v)] = child
            if last_node is not None and last_node==child:
                halo_color[v] = [0,1,0,0.5]
                weather_halo[v] = True
            else:
                weather_halo[v] = False
                halo_color[v] = [0,0,0,0]
            if isinstance(child,Node):
                shape[v] = "circle"
                recursive_add(child,v)
            else:
                shape[v] = "square"

    number_to_node = {}
    g = Graph(directed=True)
    color = g.new_vertex_property("string")
    shape = g.new_vertex_property("string")
    text = g.new_vertex_property("string")
    size = g.new_vertex_property("int")
    pen_width = g.new_vertex_property("int")
    halo_color = g.new_vertex_property("vector<double>")
    weather_halo = g.new_vertex_property("bool")
    halo_size = g.new_vertex_property("double")

    g.vp.l = halo_size
    g.vp.h = halo_color
    g.vp.g = size
    g.vp.c = color
    g.vp.s = shape
    g.vp.t = text
    g.vp.p = pen_width
    g.vp.b = weather_halo

    v = g.add_vertex()
    halo_color[v] = [0,0,0,0]
    if root==last_node:
        halo_color[v] = [0,1,0,0.5]
        weather_halo[v] = True
    if isinstance(root,Node):
        recursive_add(root,v)

    halo_size.a = 1.1
    makerturn = root.makerturn if isinstance(root,Leafnode) else root.storage.gp["m"]
    color[v] = "red" if makerturn else "blue"
    shape[v] = "square" if isinstance(root,Leafnode) else "circle"
    text[v] = ""
    size.a = 25
    return g,number_to_node

def visualize_MCTS(new_version=True):
    print("""
Instructions:
n: Node number          v: number of visits
q: Q-values             t: Total Values
m: moves                p: Priors
u: ucb                  r: get result
c: select best child for next iteration
[number]: show graph for number.
          """)
    size = 2
    game = get_graph_only_hex_game(size)
    show_game = Hex_game(size)
    # nn = get_pre_defined_mcts_model("misty-firebrand-26/11")
    nnet = get_pre_defined("policy_value",args=Namespace(**{"hidden_channels":25,"num_layers":8,"head_layers":2}))
    nn = NNetWrapper(nnet=nnet)

    if new_version:
        mcts = MCTS(game.copy(),dummy_nn,args=Namespace(cpuct=1),remove_dead_and_captured=False)
        last_hash = get_unique_hash(game.view)
    else:
        mcts = MCTS_old(game,nn.predict_for_mcts,remove_dead_captured=False)
        last_node = mcts.root
    mode = "num_visits"
    while 1:
        if new_version:
            g,number_to_state = graph_from_hash_mcts(mcts,game.copy(),to_show=mode,last_hash=last_hash)
        else:
            g,number_to_node = graph_from_root(mcts.root,to_show=mode,last_node=last_node)
        graph_draw(g,radial_tree_layout(g,g.vertex(0)),vprops={"halo_size":g.vp.l,"halo":g.vp.b,"halo_color":g.vp.h,"pen_width":g.vp.p,"shape":g.vp.s,"fill_color":g.vp.c,"text":g.vp.t,"size":g.vp.g},bg_color="black",output="mcts.pdf")
        os.system("pkill -f 'mupdf mcts.pdf'")
        os.system("nohup mupdf mcts.pdf > /dev/null 2>&1 &")
        time.sleep(0.1)
        os.system("bspc node -f west")
        while 1:
            command = input()
            if command=="n":
                mode = "numbers"
                break
            elif command=="v":
                mode = "num_visits"
                break
            elif command=="q":
                mode = "q"
                break
            elif command=="t":
                mode = "value"
                break
            elif command=="c":
                if new_version:
                    moves,probs = mcts.extract_result(Graph(game.graph),temp=0)
                else:
                    moves,probs = mcts.extract_result(0)
                action = moves[np.argmax(probs)]
                child = mcts.root.children[np.argmax(probs)]
                if isinstance(child,Leafnode):
                    print("Failed, child is leafnode")
                else:
                    mcts.next_iter_with_child(action,child.storage)
                    break

            elif command=="m":
                mode = "m"
                break
            elif command=="p":
                mode = "prior"
                break
            elif command=="u":
                mode = "ucb"
                break
            elif command=="r":
                if new_version:
                    moves,probs = mcts.extract_result(Graph(game.graph),temp=0)
                    graph_state(game.copy(),probs)
                else:
                    moves,probs = mcts.extract_result(0)
                    graph_node(mcts.root,probs)
                print(moves,probs)

            elif command=="":
                if new_version:
                    value = mcts.single_iteration(Graph(game.graph))
                    g = Node_switching_game.from_graph(mcts.leaf_graph)
                    s = get_unique_hash(g.view)
                    if s in mcts.Ps:
                        graph_state(g,mcts.Ps[s])
                    else:
                        graph_state(g)
                    last_hash = s
                else:
                    leaf,value = mcts.single_iteration()
                    if isinstance(leaf,Node):
                        graph_node(leaf)
                    if mcts.done:
                        print(mcts.extract_result(1))
                        print("MCTS is done")
                print(f"got value {value}")
                break
            else:
                if new_version:
                    state = number_to_state[int(command)]
                    g = Node_switching_game.from_graph(state)
                    graph_state(g,mcts.Ps[get_unique_hash(g.view)])
                else:
                    if int(command)==0:
                        node = mcts.root
                    else:
                        node = number_to_node[int(command)]
                    if isinstance(node,Node):
                        show_game.set_to_graph(node.storage)
                        show_game.draw_me(fname="uff.pdf")
                        os.system("nohup mupdf uff.pdf > /dev/null 2>&1 &")
                        time.sleep(0.1)
                        os.system("bspc node -f west")
                    nd = node.__dict__.copy()
                    del nd["parent"]
                    del nd["children"]

if __name__=="__main__":
    visualize_MCTS()