import os
from subprocess import call
import pandas as pd

from jellyrisk_forecaster.utils import exists
from jellyrisk_forecaster.utils import download_myocean_data, create_if_not_exists
from jellyrisk_forecaster.config import settings, BASE_DIR

TIMES_START_MEDSEA_REAN_006_004 = ['2007-05-23', '2008-05-28', '2009-05-16',
                                   '2010-05-15', '2011-06-01', '2012-06-05']
TIMES_END_MEDSEA_REAN_006_004 = ['2007-09-18', '2008-09-23', '2009-09-22',
                                 '2010-09-23', '2011-10-01', '2012-09-23']
TIMES_START_MEDSEA_FOR_006_001 = ['2013-06-03']
TIMES_END_MEDSEA_FOR_006_001 = ['2013-10-23']

DATASETS = [
    # 2007-2012
    {
        # salinity
        'service': 'http://purl.org/myocean/ontology/service/database#MEDSEA_REANALYSIS_PHYS_006_004-TDS',
        'product': 'myov04-med-ingv-sal-rean-dm',
        'times_start': TIMES_START_MEDSEA_REAN_006_004,
        'times_end': TIMES_END_MEDSEA_REAN_006_004,
        'vars': ['vosaline']
    },
    {   # temperature
        'service': 'http://purl.org/myocean/ontology/service/database#MEDSEA_REANALYSIS_PHYS_006_004-TDS',
        'product': 'myov04-med-ingv-tem-rean-dm',
        'times_start': TIMES_START_MEDSEA_REAN_006_004,
        'times_end': TIMES_END_MEDSEA_REAN_006_004,
        'vars': ['votemper']
    },
    {   # ocean currents
        'service': 'http://purl.org/myocean/ontology/service/database#MEDSEA_REANALYSIS_PHYS_006_004-TDS',
        'product': 'myov04-med-ingv-cur-rean-dm',
        'times_start': TIMES_START_MEDSEA_REAN_006_004,
        'times_end': TIMES_END_MEDSEA_REAN_006_004,
        'vars': ['vozocrtx', 'vomecrty']
    },
    # 2013
    {   # salinity
        'service': 'http://purl.org/myocean/ontology/service/database#MEDSEA_ANALYSIS_FORECAST_PHYS_006_001_a-TDS',
        'product': 'myov04-med-ingv-sal-an-fc',
        'times_start': TIMES_START_MEDSEA_FOR_006_001,
        'times_end': TIMES_END_MEDSEA_FOR_006_001,
        'vars': ['vosaline']
    },
    {   # temperature
        'service': 'http://purl.org/myocean/ontology/service/database#MEDSEA_ANALYSIS_FORECAST_PHYS_006_001_a-TDS',
        'product': 'myov04-med-ingv-tem-an-fc',
        'times_start': TIMES_START_MEDSEA_FOR_006_001,
        'times_end': TIMES_END_MEDSEA_FOR_006_001,
        'vars': ['votemper']
    },
    {   # ocean currents
        'service': 'http://purl.org/myocean/ontology/service/database#MEDSEA_ANALYSIS_FORECAST_PHYS_006_001_a-TDS',
        'product': 'myov04-med-ingv-cur-an-fc',
        'times_start': TIMES_START_MEDSEA_FOR_006_001,
        'times_end': TIMES_END_MEDSEA_FOR_006_001,
        'vars': ['vozocrtx', 'vomecrty']
    },
]

def download_historical_data(datasets, force=False):
    """
    Download historical data from MyOcean using motu-client.
    """
    folder = os.path.join(settings.DATA_FOLDER, 'MyOcean', 'Historical')
    create_if_not_exists(folder)

    for dataset in DATASETS:
        for time_start, time_end in zip(dataset['times_start'], dataset['times_end']):
            filename = '%(product)s_%(time_start)s_%(time_end)s.nc' % \
                {'product': dataset['product'],
                 'time_start': time_start,
                 'time_end': time_end}
            if not exists(filename, folder) or force:
                print('Downloading %s...' % filename)
                download_myocean_data(
                    service=dataset['service'],
                    product=dataset['product'],
                    time_start=time_start,
                    time_end=time_end,
                    folder=folder,
                    filename=filename)
            else:
                print('File %s already exists, skipping download... (use force=True to override).' % filename)

def extract_historical_data(datasets, force=False):
    """
    Extract variables values for all dates and beaches.

    If the output file is already present and force is False, skip preprocessing.
    """
    nc_folder = os.path.join(settings.DATA_FOLDER, 'MyOcean', 'Historical')

    out_folder = os.path.join(settings.DATA_FOLDER, 'MyOcean', 'Historical', 'Preprocessed')
    create_if_not_exists(out_folder)

    for dataset in DATASETS:
        for time_start, time_end in zip(dataset['times_start'], dataset['times_end']):
            nc_filename = '%(product)s_%(time_start)s_%(time_end)s.nc' % \
                {'product': dataset['product'],
                 'time_start': time_start,
                 'time_end': time_end}
            nc_filepath = os.path.join(nc_folder, nc_filename)

            for var in dataset['vars']:
                out_filename = '%(product)s_%(time_start)s_%(time_end)s-%(var)s.csv' % \
                    {'product': dataset['product'],
                     'time_start': time_start,
                     'time_end': time_end,
                     'var': var}

                if not exists(out_filename, out_folder) or force:
                    out_filepath = os.path.join(out_folder, out_filename)
                    os.chdir(os.path.join(settings.DATA_FOLDER))
                    print('Extracting to %s...' % out_filename)

                    with open(os.path.join(BASE_DIR, 'R', 'Extract_historical_data.R'), 'r') as inputfile:
                        call(["R", "--no-save", "--args",
                              time_start, time_end, nc_filepath, var, out_filepath],
                             stdin=inputfile)
                else:
                    print('\nFile %s already exists, skipping preprocessing... (use force=True to override).' % out_filename)

def preprocess_historical_data(datasets, force=False):
    """Preprocess environmental historical data from MyOcean and merge it with
    inspection vars.

    If the output file is already present and force is False, skip preprocessing.
    """
    # concatenate temporarily and merge all environmental data
    vosaline, votemper, vozocrtx, vomecrty = concatenate_historical_data(datasets)
    all_vars = votemper.merge(vosaline.merge(vozocrtx.merge(vomecrty)))

    # concatenate temporarily all inspection data
    fnames = ['SSS_2007-2010.csv', 'SSS_2011.csv', 'SSS_2012.csv', 'SSS_2013.csv']
    paths = [os.path.join(settings.DATA_FOLDER, 'DailyInspections', fname) for fname in fnames]
    observations = [pd.read_csv(path, sep='\t') for path in paths]
    obs_concated = pd.concat(observations)

    # Only consider 1 and 3 values as presence. 0 is too few jellyfishes.
    jellyfishes = ['Pel', 'Aur', 'Cot', 'Rhi', 'Chr', 'Aeq', 'Vel', 'Phys']
    for jellyfish in jellyfishes:
        obs_concated.ix[obs_concated[jellyfish] == 1, jellyfish] = 0
        obs_concated.ix[obs_concated[jellyfish] == 2, jellyfish] = 1
        obs_concated.ix[obs_concated[jellyfish] == 3, jellyfish] = 1

    # merge environmental and inspection data
    everything = all_vars.merge(obs_concated)
    filename = 'Environmental_and_Inspections_2010-2013.csv'
    out_path = os.path.join(settings.DATA_FOLDER, filename)

    if not os.path.exists(out_path) or force:
        print('Creating %s...' % filename)
        with open(out_path, 'w') as out_file:
            everything.to_csv(out_file, sep='\t', na_rep='NA', index=False)
    else:
        print('\nFile %s already exists, skipping preprocessing... (use force=True to override).' % out_path)

def concatenate_historical_data(datasets):
    """
    Concatenate temporarily all environmental data.
    """
    def get_path(dataset, var, time_start, time_end):
        folder = os.path.join(settings.DATA_FOLDER, 'MyOcean', 'Historical', 'Preprocessed')
        filename = '%(product)s_%(time_start)s_%(time_end)s-%(var)s.csv' % \
            {'product': dataset['product'],
             'time_start': time_start,
             'time_end': time_end,
             'var': var}
        return os.path.join(folder, filename)

    def concat(datasets, var):
        paths = []
        for dataset in datasets:
            for time_start, time_end in zip(dataset['times_start'], dataset['times_end']):
                paths.append(get_path(dataset, var, time_start, time_end))
        dfs = [pd.read_csv(path, sep='\t') for path in paths]
        return pd.concat(dfs)

    vosaline = concat((datasets[0], datasets[3]), 'vosaline')
    votemper = concat((datasets[1], datasets[4]), 'votemper')
    vozocrtx = concat((datasets[2], datasets[5]), 'vozocrtx')
    vomecrty = concat((datasets[2], datasets[5]), 'vomecrty')

    return (vosaline, votemper, vozocrtx, vomecrty)

def calibrate_model(force=False):
    """Calibrate model using historical data.

    If a file with the model is already present and force=False, skip."""
    folder = settings.DATA_FOLDER
    filename = 'Pnoctiluca_model.R'

    if not exists(filename, folder) or force:
        os.chdir(settings.DATA_FOLDER)
        with open(os.path.join(BASE_DIR, 'R', 'Pnoctiluca_calibrate.R'), 'r') as inputfile:
            call(["R", "--no-save"], stdin=inputfile)
    else:
        print('\nFile %s already exists, skipping modelling... (use force=True to override).' % filename)

def calibrate(force=False):
    print("\n=== Calibrating model from historical data... ===")
    download_historical_data(DATASETS, force)
    extract_historical_data(DATASETS, force)
    preprocess_historical_data(DATASETS, force)
    calibrate_model(force)

if __name__ == "__main__":
    calibrate()


#!/usr/bin/python3
import sys
sys.path.append('/home/srm/aa/PYTHON/PROJECTS/XML')
import Xmlconv;
import argparse

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('-XML_read', help="To read xml file, Enter file name.")
    parser.add_argument('-Write_EXCEL', help="To Write excel file Enter file name.")
    parser.add_argument('-Write_JSON', help="To write json file Enter file name.")
    parser.add_argument('-Write_DBI', help="To write mysql dbi table Enter file name.")
    parser.add_argument('-Write_CSV', help="To write csv file Enter file name.")
    args = parser.parse_args()

    if args.XML_read:
       aa = Xmlconv.parseXML(args.XML_read)
      #print(aa)

    if args.Write_EXCEL:
        Xmlconv.Write_EXCEL(args.Write_EXCEL,aa)

    if args.Write_JSON:
        Xmlconv.Write_JSON(args.Write_JSON,aa)

    if args.Write_CSV:
        Xmlconv.Write_CSV(args.Write_CSV,aa)

    if args.Write_DBI:
        Xmlconv.Write_DBI(args.Write_DBI,aa)


import PySimpleGUI as sg
import pandas as pd
import sys
import traceback
from datetime import datetime

import utils
from lib import lib_sys
from logger import log_debugger

def add_task(project, task, content, name_assignee, team, start_date, end_date):
    try:
        df = pd.DataFrame(
            columns = ['DATE_MODIFIED', 'PROJECT', 'TASK', 'CONTENT', 'STATUS',
                'ASSIGNER', 'ASSIGNEE', 'TEAM', 'ISSUE', 'SOLUTION',
                'SUPERVISOR_COMMENT', 'START_DATE', 'END_DATE', 'NOTE']
        )

        # Replacing data at each cell of the new row
        df['DATE_MODIFIED'] = [datetime.now()]
        df['PROJECT'] = project
        df['TASK'] = task
        df['CONTENT'] = content
        df['STATUS'] = 'Open'

        df['ASSIGNER'] = utils.username
        df['ASSIGNEE'] = name_assignee
        df['TEAM'] = team
        df['ISSUE'] = 'None'
        df['SOLUTION'] = 'None'

        df['SUPERVISOR_COMMENT'] = 'None'
        df['START_DATE'] = datetime.strptime(start_date, '%d-%b-%y')
        df['END_DATE'] = datetime.strptime(start_date, '%d-%b-%y')
        df['NOTE'] = 'None'

        df.to_sql(utils.db_task, utils.ENGINE, if_exists = 'append', index = False)
        return 'Add task successfully.'

    except Exception as e:
        return 'Add task failed!'

def make_table_task(name):
    df = pd.read_sql(
        "select * from {} where ASSIGNEE like '{}'".format(utils.db_task, name), utils.ENGINE)
    df.columns = [x.upper() for x in df.columns]

    return df

def create_window_task(data, header):
    sg.theme('LightGreen')
    if sys.platform != 'win32':
        sg.set_options(font = ('Helvetica', 13))

    layout = [
        [sg.Table(
            key = '-TABLE-',
            values = data,
            headings = header,
            max_col_width = 25,
            auto_size_columns = True,
            display_row_numbers = True,
            justification = 'right',
            num_rows = 10,
            alternating_row_color = 'white',
            row_height = 35
        )],
        [
            sg.Button('Choose'),
            sg.Button('Modify')
        ]
    ]

    window = sg.Window(
        'List of Tasks',
        layout,
        keep_on_top = True
    )

    return window

def get_task(name = utils.username):
    df_task = make_table_task(name)
    data = df_task.values.tolist()
    header = df_task.columns.tolist()
    window = create_window_task(data, header)
    task = ''

    while True:
        try:
            event, values = window.read()
            if event == 'Choose':
                task_index = values.get('-TABLE-')[0]
                df_task.loc[df_task['STATUS'] == 'In Progress', 'STATUS'] = 'Pending'
                df_task.loc[task_index, 'STATUS'] = 'In Progress'

                utils.ENGINE.execute("delete from {} where ASSIGNEE like '{}'".format(utils.db_task, name))
                df_task.to_sql(utils.db_task, utils.ENGINE, if_exists = 'append', index = False)
                task = df_task.loc[task_index, 'TASK']

                window.Close()

            elif event == 'Modify':
                print(event, values)

            if event == sg.WIN_CLOSED:
                window.close()
                break

        except UnicodeDecodeError:
            pass

        except KeyboardInterrupt:
            pass

        except:
            log_debugger.warning(traceback.format_exc())
            window.Close()
            break

    return "'{}': Task is selected.".format(task)

def create_window_todo():
    sg.theme('TanBlue')
    if sys.platform != 'win32':
        sg.set_options(font = ('Helvetica', 15))

    layout = [
        [sg.Text('Today Outcomes')],
        [sg.Multiline(key = '-OUTCOMES-', size = (45, 5))],
        [sg.Text('To Do')],
        [sg.Multiline(key = '-TODO-', size = (45, 5))],
        [sg.Submit(key = '-SUBMIT-')]
    ]

    window = sg.Window(
        'To Do List',
        layout,
        keep_on_top = True,
        finalize = True
    )

    return window

def add_todo():
    window = create_window_todo()

    while True:
        try:
            event, values = window.read(timeout = 100)
            if event == sg.TIMEOUT_KEY:
                tp_outcomes = values['-OUTCOMES-']
                tp_todo = values['-TODO-']
                pass

            if len(tp_outcomes.replace('\n','')) > 0 and len(tp_todo.replace('\n','')) > 0 and event == '-SUBMIT-':
                if int(utils.server_status) == 0:
                    df_todo = pd.DataFrame([{
                        'DATETIME': datetime.now(),
                        'TIME': datetime.now().strftime('%H:%M:%S'),
                        'NAME': utils.username,
                        'OUTCOMES': tp_outcomes,
                        'TODO': tp_todo
                    }])
                    print(df_todo)
                    df_todo.to_sql(utils.db_todo, utils.ENGINE, if_exists = 'append', index = False)

                window.Hide()
                window.Close()
                return 'Done'

            if event == sg.WIN_CLOSED:
                window.Close()
                break

        except UnicodeDecodeError:
            pass

        except KeyboardInterrupt:
            pass

        except:
            log_debugger.warning(traceback.format_exc())
            window.Close()
            break

def create_window_recommendations():
    sg.theme('TanBlue')
    if sys.platform != 'win32':
        sg.set_options(font = ('Helvetica', 15))

    layout = [
        [sg.Text('3 Recommendations')],
        [sg.Multiline(key = '-RECOMMENDATION-', size = (45, 10))],
        [sg.Submit(key = '-SUBMIT-')]
    ]

    window = sg.Window(
        'Recommendations List',
        layout,
        keep_on_top = True,
        finalize = True
    )

    return window

def add_recommendations():
    window = create_window_recommendations()

    while True:
        try:
            event, values = window.read(timeout = 100)
            if event == sg.TIMEOUT_KEY:
                tp_recommendation = values['-RECOMMENDATION-']
                pass
            if len(tp_recommendation.replace('\n','')) > 0 and event == '-SUBMIT-':
                if int(utils.server_status) == 0:
                    df_todo = pd.DataFrame([{
                        'DATETIME': datetime.now(),
                        'TIME': datetime.now().strftime('%H:%M:%S'),
                        'NAME': utils.username,
                        'RECOMMENDATION': tp_recommendation,
                    }])
                    print(df_todo)
                    df_todo.to_sql(utils.db_recommendation, utils.ENGINE, if_exists = 'append', index = False)

                window.Hide()
                window.Close()
                return 'Done'

            if event == sg.WIN_CLOSED:
                window.Close()
                break

        except UnicodeDecodeError:
            pass

        except KeyboardInterrupt:
            pass

        except:
            log_debugger.warning(traceback.format_exc())
            window.Close()
            break


fhandle=open('mbox-short.txt','r')
counts=dict()
for line in fhandle:
    line=line.rstrip()
    if len(line) < 1 :
        continue
    if not line.startswith('From:'):
        continue
    words=line.split()
    emailad=words[1]
    emails=emailad.split()
    for email in emails:
        counts[email]=counts.get(email,0) + 1

bigemail=None
bigcount=None
for email,count in counts.items():
    if bigemail is None or count>bigcount:
        bigemail=email
        bigcount=count
print(bigemail,bigcount)


# Time complexity: O(N * M) (M = number of colors)
# Space complexity: O(N * M)
#Compiled on leetcode: Yes

class Solution:
    def minCost(self, costs: List[List[int]]) -> int:
        rows = len(costs)
        if rows == 0:
            return 0
        columns = 3
        dp = [[0 for i in range(columns)] for j in range(rows)]
        dp[0] = costs[0]
        for i in range(1, rows):
            for j in range(columns):
                if j == 0:
                    dp[i][j] = costs[i][j] + min(dp[i - 1][j + 1], dp[i - 1][j + 2])
                elif j == 1:
                    dp[i][j] = costs[i][j] + min(dp[i - 1][j - 1], dp[i - 1][j + 1])
                else:
                    dp[i][j] = costs[i][j] + min(dp[i - 1][j - 1], dp[i - 1][j - 2])
        return min(dp[-1])


#Pendulumの問題を強化学習で解く
from datetime import datetime
import os
import shutil
from statistics import median, mean

from actor_critic_agent import AgentCreater
from actor_critic_agent.memory import Memory

from pendulum import Environment
from pendulum.models import ValueNet, PolicyNet

from test_log import TestReport, DisplayTestLog, FileTestLog, TestLogs

max_episodes = int(1e5)
test_interval = 500
test_repeat = 10

#ログをファイルに残すか？
leaving_logs = True

#環境を作成する
environment = Environment()

#モデルを作成する
v_model = ValueNet()
pi_model = PolicyNet()

#エージェントを作成する
parameters_filename = "pendulum_parameters.json"
creater = AgentCreater(parameters_filename)
agent = creater.create(v_model, pi_model)

if leaving_logs:
    #ログフォルダを作成する
    if not os.path.exists("./log/"):
        os.mkdir("./log/")
    #if

    if not os.path.exists("./log/pendulum/"):
        os.mkdir("./log/pendulum/")
    #if

    log_folder = "./log/pendulum/" + datetime.now().strftime('%Y%m%d%H%M%S')
    os.mkdir(log_folder)

    #テストログファイル
    test_log_filename =  log_folder + "/test_log.txt"

    #テストログを画面に表示&ファイルに保存する
    test_logs = [DisplayTestLog(), FileTestLog(test_log_filename)]

    #パラメーターファイルをコピーする
    shutil.copy(parameters_filename, log_folder+"/"+parameters_filename)

    #モデル(のソースコード)をコピーする
    shutil.copy("./pendulum/models.py", log_folder+"/models.py")

else:
    #テスト結果をディスプレイに表示する
    test_logs = [DisplayTestLog(),]
#if-else

#テストログ表示
test_log = TestLogs(test_logs)

max_steps = environment.max_steps

#試行を繰り返して訓練する
for t in range(1, max_episodes+1):
    state = environment.reset()

    for step in range(1, max_steps+1):
        #試行する
        action = agent.action(state)
        (state_dash, reward, done, info) = environment.step(action)
        agent.gain_experience(state, action, reward, state_dash, done, info)

        #経験がたまったら学習する
        if agent.is_experienced():

            #テストの周期にあたる場合は訓練誤差を計算する
            estimate_loss = (t%test_interval == 0)

            (training_V_loss, training_pi_loss, training_pi_entropy) = agent.train(estimate_loss=estimate_loss)
            agent.reset_experience()
        #if

        state = state_dash
    #for step in range(1, max_steps+1)

    #テストする
    if t % test_interval == 0:
        total_rewards = []
        steps = []

        #テスト用のメモリー
        test_memory = Memory(test_repeat*max_steps)

        #テストを複数エピソードを繰り返す
        for _ in range(test_repeat):
            total_reward = 0

            state = environment.reset()
            for step in range(1, max_steps+1):
                #行動する
                action = agent.action(state)
                (state_dash, reward, done, info) = environment.step(action)

                test_memory.append(state, action, reward, state_dash, done)
                total_reward += reward

                state = state_dash
            #for step in range(1, max_steps+1)

            total_rewards.append(total_reward)
            steps.append(step)
        #for _ in range(test_repeat):

        #収益のmedianと最小値
        median_total_reward = median(total_rewards)
        worst_total_reward = min(total_rewards)

        #ステップ数
        step = median(steps)

        #テスト時の誤差を計算する
        experiences = test_memory.refer()
        (test_V_loss, test_pi_loss, test_pi_entropy) = agent.estimate_loss_from_experience(experiences)

        #テスト結果を表示する
        report = TestReport(
                            t,
                            step,
                            median_total_reward, worst_total_reward,
                            training_V_loss, training_pi_loss, training_pi_entropy,
                            test_V_loss, test_pi_loss, test_pi_entropy
                        )
        test_log.print(report)

        #モデルを保存する
        if leaving_logs:
            episode_string = "Episode-" + str(t).zfill(10)
            model_filename = log_folder +"/" +episode_string + ".model"
            agent.save(model_filename)
        #if

    #if t % test_interval == 0:
#for t in range(1, max_episodes):

environment.close()


# -*- coding: utf-8 -*-
from __future__ import division
import math

n1= input('Digite n1: ')
n2= input('Digite n2: ')

cont1=0
while n1>1:
    cont1= cont1+1
    n1= n1/10

cont2=0
while n2!=int(n2):
    cont2= cont2+1
    n2= n2*10

print(cont1)
print(cont2)


# uncompyle6 version 3.7.4
# Python bytecode 2.6 (62161)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: build/bdist.linux-i686/egg/ice/control/repl/session.py
# Compiled at: 2010-08-27 06:32:04
import os
from zope.interface import implements
from zope.component import getUtilitiesFor
from interpreter import Interpreter
from interfaces import ISession

class Session:
    implements(ISession)
    input_buffer = ''
    history = []
    h_max = 10

    def __init__(self, context):
        self.interpreter = Interpreter({'__name__': '__console__', '__doc__': None,
           'context': context})
        bootstrap = file(os.path.join(os.path.dirname(__file__), 'bootstrap.py'))
        for line in bootstrap.readlines():
            self.run(line)

        self.history = [
         '']
        return

    def update_history(self, source):
        try:
            self.history.remove(source)
        except ValueError:
            pass

        if source:
            self.history.insert(0, source)
        if len(self.history) > self.h_max:
            self.history = self.history[:self.h_max]

    def run(self, source):
        self.update_history(source)
        self.input_buffer += source
        result = self.interpreter.runsource(self.input_buffer)
        if result:
            self.input_buffer += '\n'
        else:
            self.input_buffer = ''
        return (
         result, self.interpreter.get_output())

    def get_history(self):
        return self.history


from cassandra import cqlengine
class TableIterator(object):
    """
    Iterates over a Cassandra table defined by a cqlengine model class using query paging in order to pull back chunks
    of data that have `blocksize` number of records.
    Can optionally provide kwargs which are used as where clause filters. These kwargs must be columns on the model
    which are indexed.
    :param model_class: The cqlengine model object that defines the table you want to iterate over.
    :type model_class: An instance of cqlengine.Model.
    :param blocksize: The number of results you want to pull back with each paging query. Can be used to tune the
                        performance of the iteration depending on the characteristics of the table.
    :type blocksize: integer
    :param where_filters: Keyword arguments can be passed to the iterator and these will be used as filter parameters
                        when querying the database for pages of data.
    :type where_filters: **kwargs
    :return: an iterator over the a collection of objects of type model_class.
    """

    def __init__(self, model_class, blocksize=10000, **where_filters):
        self.model_class = model_class

        # Pull the keys of the model class for convenient reference.
        self.partition_keys = model_class._partition_keys
        self.clustering_keys = model_class._clustering_keys
        self.blocksize = blocksize
        self.where_filters = where_filters

    @classmethod
    def generate_where_clause_key(cls, column_name, clause_condition):
        """
        Takes the name of a primary key column and a condition ('gt', 'lt', 'eq') and creates a where clause key that
        can be used with a cqlengine .objects() descriptor to filter based on that condition.
        :param key_name: The name of the model column (primary key) for which you want to generate a where clause.
        :type key_name: The string representation of the column name.
        :param clause_condition: The conditional operator you want to filter by.
        :type clause_condition: A string that is a valid cqlengine conditional operator ('gt', 'lt', 'eq')
        :return: A string of the form "{my_column_name}__{my_clause_condition}".
        """
        return "{}__{}".format(column_name, clause_condition)

    @classmethod
    def get_paging_where_clause_key(cls, primary_key_column):
        """
        Get a where clause key value that can be used to page through the primary key column.
        :param primary_key_column: A primary key column class you want a key to page over.
        :type primary_key_column:  An class that inherits from cqlengine.Column.
        :return: A string of the format "{column_name}__{where_condition}" which will page data from that column in the
                direction defined by the clustering order of that column.
                For example, if I have a clustering key named `my_cluster_key` which has a descending clustering order,
                this function will return the key 'my_cluster_key__lt' which can be used to page over the value of
                my_cluster_key in it's default order.
        """
        condition_lookup = {"asc": "gt", "desc": "lt"}
        clustering_order = getattr(primary_key_column, "clustering_order") or "asc"
        clause_condition = condition_lookup[clustering_order.lower()]
        return cls.generate_where_clause_key(primary_key_column.column_name, clause_condition)

    def get_next_query_set(self, previous_object):
        """
        Takes a cqlengine model object instance and treats that object as the current cursor into a Cassandra table
        generating a cqlengine query object which will page in the set of results immediately following
        `previous_object` according to Cassandra partition tokens and clustering order.
        :param previous_object: The last object fetched by the previous paging query. Can also be viewed as the cursor
                    location for this table iteration.
        :type previous_object: A cqlengine model object instance.
        :return: A cqlengine QuerySet object that will return the objects immediately following `previous_object` in the
                Cassandra table.
        """
        prev_partition_key_vals = {}
        prev_clustering_key_vals = {}

        # Pull all of the key values off of previous_object
        for p_key_name, _ in self.partition_keys.items():
            prev_partition_key_vals[p_key_name] = getattr(previous_object, p_key_name)
        for c_key_name, _ in self.clustering_keys.items():
            prev_clustering_key_vals[c_key_name] = getattr(previous_object, c_key_name)

        # Copy the clustering keys dict since we want to use the values it contains as **kwargs to a QuerySet. We need
        # to alter the values without clobbering the original values.
        cluster_where_clause = dict(prev_clustering_key_vals.items())

        # Iterator over the ordered clustering keys in reverse order.
        for c_key_name, c_key_col in reversed(self.clustering_keys.items()):
            # Drop the equals clause for the current clustering key because we want a paging conditional ('gt' or 'lt').
            del cluster_where_clause[c_key_name]

            # Generate a paging clause for this clustering key that we will use as a where clause filter.
            new_where_key = self.get_paging_where_clause_key(c_key_col)
            cluster_where_clause[new_where_key] = prev_clustering_key_vals[c_key_name]

            # Generate our new where clause consisting of the current partition, our paging clustering conditions and
            # any where_filters that were originally handed to TableIterator.
            where_clause = dict(prev_partition_key_vals.items() + cluster_where_clause.items() + self.where_filters.items())

            current_query = self.model_class.objects(**where_clause).limit(self.blocksize)

            # TODO: Can we optimize to return results from this function rather than doing garbage query round trip?
            if current_query.first():
                # This query returns objects, so it's a valid page and we want to use it.
                return current_query
            else:
                # This query returned nothing so we have exhausted the clustering key we are currently looking at.
                # Drop the clause for this clustering key and continue to the next one.
                del cluster_where_clause[new_where_key]

        # We made it through testing all of the clustering key values and got no results so we have exhausted the
        # current partition we are looking at.

        # Generate the partition key token for the last seen object.
        token = cqlengine.functions.Token(previous_object.pk)

        # Create a where clause for the partition key token.
        pk_token_where = self.generate_where_clause_key('pk__token', 'gt')
        partition_key_clause = {pk_token_where: token}

        where_clause = dict(partition_key_clause.items() + self.where_filters.items())

        query = self.model_class.objects(**where_clause).limit(self.blocksize)

        return query

    def __iter__(self):
        """
        Returns an iterator over the objects that exist in the table passed into __init__.
        """

        done_iterating = False
        query = self.model_class.objects(**self.where_filters).limit(self.blocksize)

        while not done_iterating:
            previous_object = None

            for obj in query:
                previous_object = obj
                yield obj

            if not previous_object is None:
                query = self.get_next_query_set(previous_object)
            else:
                done_iterating = True


import numpy as np
import matplotlib.pyplot as plt
from configparser import ConfigParser
import scipy.io as sio
from tqdm import tqdm
from scipy.fftpack import fft
from visualizer import multi_slice_viewer

config = ConfigParser()
config.read("config.ini")
cfg = config['DEFAULT']
files_path = cfg['files_path']

data1=sio.loadmat(files_path + 'fx1_XTRAIN.mat')
data2=sio.loadmat(files_path + 'Y_train.mat')
#X_pred = data1['X_PRED']
X_train = data1['X_TRAIN']  # (110, 80, 80)

Y_train = data2['Y_train']  # (110, 1024, 768, 1, 3)

Y0 = np.array([Y_train[n,:,:,0,0] - Y_train[0,:,:,0,0] for n,_ in enumerate(Y_train)])
Y1 = np.array([Y_train[n,:,:,0,1] - Y_train[0,:,:,0,1] for n,_ in enumerate(Y_train)])
print(Y0.shape)
#multi_slice_viewer(Y0)

#multi_slice_viewer(Y1)

#plt.plot([np.std(n) for n in Y0])
#plt.show()
"""for i in range(10):
    idx = i*3+50
    plt.figure()
    plt.plot(Y0[i+50,530,:])
    plt.title(str(idx))
    plt.show()"""
B=[]
for n in Y0:
    A= []
    for d in n:
        A.append(fft(d)[3:384])
    B.append(A)
B = np.array(B, dtype=np.float32)
multi_slice_viewer(B) # only if 2d array
#multi_slice_viewer(np.array(B, dtype=np.float32))
#multi_slice_viewer(Y0)
"""plt.plot([np.sum(a) for a in B])
plt.show()"""

"""try:
    X = np.load(files_path+"Xxxx.npy")
    print("loaded X from memory")
except IOError:
    print("standardizing and saving X")
    X = (X_train - np.mean(X_train)) / np.std(X_train)
    np.save(files_path + "X.npy", X)"""

"""try:
    Y = np.load(files_path + "Yxxx.npy")
    print("loaded Y from memory")
except IOError:
    Z = np.zeros((110, 1024, 768, 2), dtype = np.float32)
    print("standardizing and saving Y")
    for idz, z in tqdm(enumerate(Y_train)):
        Z[idz,:,:,0] = np.transpose(np.transpose(z[:,:,0,0]) - np.arange(1024))
        #Z[idz,:,:,1] = z[:,:,0,1] - np.arange(768)
    #Y = Z/np.std(Z)
    #del Z
    #np.save(files_path+"Y.npy",Y)"""

#g1024 = [np.arange(1024) for n in Y_train[0,0,:,0,0]]
#g768 = [np.arange(768) for n in Y_train[0,:,0,0,0]]
#plt.imshow(Y_train[0,:,:,0,0] - np.arange(1024) )
#plt.imshow(Y_train[10,:,:,0,1] - Y_train[0,:,:,0,1])
#plt.show()
#plt.imshow(Y_train[10,:,:,0,0] - Y_train[0,:,:,0,0])  #np.transpose(g1024)-
#plt.show()


#!/usr/bin/env python3

import asyncio
import yaz

class Coroutine(yaz.Plugin):
    @yaz.task
    async def do_one(self, sleep: float):
        await asyncio.sleep(sleep)
        return sleep

    @yaz.task
    async def do_many(self, count: int, sleep: float):
        return await asyncio.gather(*[self.do_one(sleep) for _ in range(count)])

class TestTask(yaz.TestCase):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.caller = self.get_caller([Coroutine])
        self.loop = asyncio.get_event_loop()
        self.sleep = 0.1
        self.delta = self.sleep * 0.25

    def test_010_one(self):
        """Should call one task asynchronously"""
        start = self.loop.time()
        self.assertEqual(self.sleep, self.caller("do-one", str(self.sleep)))
        self.assertAlmostEqual(self.sleep, self.loop.time() - start, delta=self.delta)

    def test_020_many(self):
        """Should call multiple tasks asynchronously"""
        start = self.loop.time()
        self.assertEqual([self.sleep for _ in range(10)], self.caller("do-many", "10", str(self.sleep)))
        self.assertAlmostEqual(self.sleep, self.loop.time() - start, delta=self.delta)

if __name__ == "__main__":
    yaz.main()


import logging
from datetime import datetime, timedelta
from typing import Dict, List

import pandas as pd

from business.helpers import getTimeBlocks
from business.model.asset import Asset
from business.model.timeframe import TimeFrame
from business.modules.asset_bl import AssetBL

# create logger
log = logging.getLogger("CellarLogger")

def updateFutures(
    asset: Asset, timeframe: TimeFrame, bl: AssetBL
) -> List[Dict]:
    result = []
    for cd in asset.contractDetails:

        lastTradeDateTime = datetime.strptime(
            cd.contract.lastTradeDateOrContractMonth, "%Y%m%d"
        )
        now = datetime.now()

        if lastTradeDateTime > now:

            # localSymbol = cd.contract.localSymbol
            localSymbol = f"{cd.contract.localSymbol}-{cd.contract.lastTradeDateOrContractMonth}"
            symbolData = bl.getHistoricalDataFromDB(localSymbol, timeframe)

            if symbolData is not None:

                lastDateTime = symbolData.tail(1).index[0]

                if now > lastDateTime:
                    result.append(
                        {
                            "contract": cd.contract,
                            "from": lastDateTime,
                            "to": now,
                        }
                    )
            else:
                result.append(
                    {
                        "contract": cd.contract,
                        "from": datetime.strptime("19860101", "%Y%m%d"),
                        "to": now,
                    }
                )

        else:
            log.info(
                f" SKIPPED - {cd.contract.localSymbol} - {cd.contract.lastTradeDateOrContractMonth}"
            )

    return result

def downloadFutures(asset: Asset, blockSize: int) -> List[Dict]:
    result = []

    for cd in asset.contractDetails:

        lastTradeDateTime = datetime.strptime(
            cd.contract.lastTradeDateOrContractMonth, "%Y%m%d"
        )
        now = datetime.now()

        if (
            lastTradeDateTime > datetime.strptime("19860101", "%Y%m%d")
            and lastTradeDateTime < now
        ):
            result.append(
                {
                    "contract": cd.contract,
                    "from": lastTradeDateTime - timedelta(days=blockSize),
                    "to": lastTradeDateTime,
                }
            )
        elif lastTradeDateTime >= now:
            result.append(
                {
                    "contract": cd.contract,
                    "from": now - timedelta(days=blockSize),
                    "to": now,
                }
            )

    return result

def updateStock(
    asset: Asset, timeframe: TimeFrame, histData: pd.DataFrame
) -> List[Dict]:
    result = []

    contract = asset.contractDetails[0].contract
    # symbolData = bl.getHistoricalDataFromDB(asset.symbol, timeframe)

    if histData is not None:

        lastDateTime = histData.tail(1).index[0]
        now = datetime.now()

        if now > lastDateTime:
            result.append(
                {"contract": contract, "from": lastDateTime, "to": now}
            )

    return result

def downloadStock(asset: Asset, blockSize: int) -> List[Dict]:
    result = []

    contract = asset.contractDetails[0].contract

    timeBlocks = getTimeBlocks(
        datetime.strptime("19860101", "%Y%m%d"), datetime.now(), blockSize
    )

    for timeBlock in timeBlocks:
        result.append(
            {"contract": contract, "from": timeBlock[0], "to": timeBlock[1],}
        )

    return result


import torch
import torch.nn as nn
import numpy as np
from models import PartAlign
from utils.config import Config
from datasets import CUB200
from torchvision import transforms
from utils import array_tool as at
from utils import queue_tool as qt

class PartAlignManager(object):
    def __init__(self, options, path):
        self.options = options
        self.path = path
        self.cam_weight = np.load('cam_weight.npy')

        self.net = PartAlign(
            Config.input_channel,
            Config.roi_size,
            Config.spatial_scale,
            self.cam_weight,
            Config.n_class,
            Config.n_features
        ).to(self.options['device'])

        print(self.net)
        self.cls_criterion = nn.CrossEntropyLoss()
        self.solver = torch.optim.SGD(self.net.parameters(), lr=self.options['base_lr'],
                                     momentum=0.9, weight_decay=self.options['weight_decay'])
        self.scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
            self.solver, mode='max', factor=0.1, patience=3, verbose=True,
            threshold=1e-4)

        transform_train_list = [
            transforms.Resize(448),
            transforms.RandomHorizontalFlip(),
            transforms.RandomCrop(size=448),
            transforms.ToTensor(),
            transforms.Normalize(mean=(0.485, 0.456, 0.406),
                                 std=(0.229, 0.224, 0.225))
        ]
        transforms_test_list = [
            transforms.Resize(448),
            transforms.CenterCrop(448),
            transforms.ToTensor(),
            transforms.Normalize(mean=(0.485, 0.456, 0.406),
                                 std=(0.229, 0.224, 0.225))
        ]
        train_data = CUB200(
            self.path['cub200'], train=True,
            transform=transforms.Compose(transform_train_list))
        test_data = CUB200(
            self.path['cub200'], train=False,
            transform=transforms.Compose(transforms_test_list))

        self.train_loader = torch.utils.data.DataLoader(
            train_data, batch_size=self.options['batch_size'],
            shuffle=True, num_workers=4, pin_memory=True
        )
        self.test_loader = torch.utils.data.DataLoader(
            test_data, batch_size=16,
            shuffle=False, num_workers=4, pin_memory=True
        )

    def train(self):
        print('Training!')
        if torch.cuda.device_count() > 1:
            print('More than one gpu')
        best_acc = 0.0
        best_epoch = 0
        print('Epoch\tTraining Loss\tTesting Loss:')
        for epoch in range(self.options['epochs']):
            num_total = 0
            num_acc = 0
            for imgs, labels in self.train_loader:
                loss = list()
                imgs = imgs.to(self.options['device'])
                labels = labels.to(self.options['device'])
                print('labels are')
                print(labels)
                feature_maps, part, rois, cam_out, cam_weight, roi_poolings = self.net(imgs)
                order_decode = self._order_decoder(part)
                order_softmax_gt = self._order_softmax_gt(order_decode, roi_poolings, labels)
                order_softmax_loss = self.cls_criterion(part, at.totensor(order_softmax_gt))
                #print(labels)
                cam_out =  cam_out.squeeze()
                cam_loss = self.cls_criterion(cam_out, labels)
                #add roi loc loss diversity cam_iou
                order_softmax_gt_decode = [list(map(int, at.part_dict[order_softmax_gt[i_order]].split(','))) for i_order in range(len(order_softmax_gt))]
                part_fc_list = list()
                roi_poolings = roi_poolings.view(roi_poolings.size(0), 4, -1)
                #print(roi_poolings)
                for i_order in range(len(order_softmax_gt_decode)):
                    #print(roi_poolings[i_order])
                    part_fc_per_img_list = list()
                    order = order_softmax_gt_decode[i_order]
                    for i_idx in range(len(order)):
                        part_fc_per_img_list.append(self.net.part_fc[order[i_idx]](roi_poolings[i_order][i_idx]))
                    assert len(part_fc_per_img_list) == 4
                    part_fc_list.append(torch.cat(tuple(part_fc_per_img_list)))
                part_fc_input = torch.cat((part_fc_list)).contiguous().view(roi_poolings.size(0), -1)
                roi_poolings = roi_poolings.view(roi_poolings.size(0), -1)
                all_part_fc = self.net.all_part_fc(part_fc_input)
                cls_loss = self.cls_criterion(all_part_fc, labels)
                loss = cls_loss+cam_loss+order_softmax_loss
                print(loss)
                loss.backward()
                #_, prediction = torch.max(all_part_fc, 1)

    def _accuracy(self):
        num_total = 0
        num_acc = 0
        self.net.eval()
        with torch.on_grad():
            for imgs, labels in self.test_loader:
                imgs = imgs.to(self.options['device'])
                labels = labels.to(self.options['device'])

    def _order_decoder(self, part):
        _, prediction = torch.max(part, 1)
        #part_numpy = at.tonumpy(part).squeeze()
        prediction_np = at.tonumpy(prediction).squeeze()
        part_decode = list()
        for i in range(len(prediction_np)):
            part_decode.append(at.part_dict[prediction_np[i]])
        return part_decode

    def _order_softmax_gt(self, order_decode, roi_poolings, gt_labels):
        gt_labels_np = at.tonumpy(gt_labels).astype(np.int)
        roi_poolings_np = at.tonumpy(roi_poolings)
        order_gt = np.asarray(gt_labels_np)
        print('gt_labels size(0) is {}'.format(gt_labels.size(0)))
        #print(gt_labels_np)
        for i in range(gt_labels.size(0)):
            #print(i)
            queue_empty, queue_features = self.net.queue.get_features(gt_labels_np[i])
            if queue_empty:
                print('query_empty')
                self.net.queue.put(gt_labels_np[i], roi_poolings_np[i])
                order_gt[i] = at.order_dict[order_decode[i]]
            else:
                roi_per_image = roi_poolings_np[i].reshape(4, -1)
                best_sim = 0.0
                best_order = -1
                for i_order in range(Config.n_order):
                    reorder = list(map(int, at.part_dict[i_order].split(',')))
                    roi_reorder = roi_per_image[reorder, :]     #reorder
                    sim = 0.0
                    for i_feature in range(Config.n_features):
                        queue_feature = queue_features[i_feature].reshape(4, -1)
                        for i_roi in range(4):
                            #print(roi_reorder[i_roi])
                            #print(queue_feature[i_roi])
                            sim += qt.cosine_similarity(roi_reorder[i_roi], queue_feature[i_roi])
                    if best_sim < sim:
                        best_sim = sim
                        best_order = i_order
                assert best_order != -1
                order_gt[i] = best_order
                if best_sim>0.6*4:
                    print(gt_labels_np[i])
                    print(gt_labels_np)
                    print('Execute order upgrating for class {}'.format(gt_labels_np[i]))
                    self.net.queue.put(gt_labels_np[i], roi_poolings_np[i])
        return order_gt

def _smooth_l1_loss(x, t, in_weight, sigma):
    sigma2 = sigma ** 2
    diff = in_weight * (x - t)
    abs_diff = diff.abs()
    flag = (abs_diff.data < (1. / sigma2)).float()
    y = (flag * (sigma2 / 2.) * (diff ** 2) +
         (1 - flag) * (abs_diff - 0.5 / sigma2))
    return y.sum()

def _loc_loss(pred_loc, gt_loc, gt_label, sigma):
    in_weight = torch.zeros(gt_loc.shape).cuda()
    # Localization loss is calculated only for positive rois.
    # NOTE:  unlike origin implementation,
    # we don't need inside_weight and outside_weight, they can calculate by gt_label
    in_weight[(gt_label > 0).view(-1, 1).expand_as(in_weight).cuda()] = 1
    loc_loss = _smooth_l1_loss(pred_loc, gt_loc, in_weight.detach(), sigma)
    # Normalize by total number of negtive and positive rois.
    loc_loss /= ((gt_label >= 0).sum().float()) # ignore gt_label==-1 for rpn_loss
    return loc_loss


import pyodbc
from Datoss.Conexion import Conexion
from Modelo.Cliente import Cliente
from Modelo.UnidadTrans import UnidadTrans

class ClientesDAO:
    db = None
    def __init__(self):
        conx = Conexion()
        self.db = conx.getDB()
    def obtenerClientes(self):
        sql = "select idCliente,nombre,razonSocial,limiteCredito,direccion," \
              "codigoPostal,rfc,telefono,email,tipo,idCiudad from Ventas.Clientes where estatus = 'A'"
        lista = []
        try:
            cursor=self.db.cursor()
            cursor.execute(sql);
            data = cursor.fetchall()
            for dato in data:
                fila = {"id":dato[0],"nombre":dato[1],"razon":dato[2],"limite":dato[3],"direccion":dato[4],
                        "codigo":dato[5],"rfc":dato[6],"tel":dato[7],"email":dato[8],"tipo":dato[9],"ciudad":dato[10]}
                lista.append(fila)
            cursor.close()
            self.db.close()
        except pyodbc.Error as error:
            print(error)
        return lista
    def insertarCliente(self,cliente):
        sql = 'insert Ventas.Clientes (idCliente,nombre,razonSocial,limiteCredito,direccion,codigoPostal,' \
              'rfc,telefono,email,tipo,idCiudad,estatus) values (?,?,?,?,?,?,?,?,?,?,?,?)'
        try:
            Values = [cliente.idCliente,cliente.nombre,cliente.razonSocial,cliente.limiteCredito,cliente.direccion,
                      cliente.codigoPostal,cliente.rfc,cliente.telefono,cliente.email,cliente.tipo,cliente.idCiudad]
            cursor=self.db.cursor()
            cursor.execute(sql,Values)
            self.db.commit()
            cursor.close()
            self.db.close()
        except pyodbc.Error as error:
            print(error)
    def ultimoID(self):
        sql="select max(idCliente)+1 id from Ventas.Clientes"
        id=1
        try:
            cursor=self.db.cursor()
            cursor.execute(sql)
            rs=cursor.fetchone()
            id=rs[0]
            cursor.close()
            self.db.close()
        except pyodbc.Error as e:
            print(e)
        return id
    def consultaIndividual(self,id):
        sql = 'select idCliente,nombre,razonSocial,limiteCredito,direccion,' \
              'codigoPostal,rfc,telefono,email,tipo,idCiudad from Ventas.Clientes where idCliente=(?)'
        c=None
        try:
            cursor = self.db.cursor()
            Values = [id]
            cursor.execute(sql,Values)
            rs = cursor.fetchone()
            c = Cliente(rs[0], rs[1], rs[2], rs[3],rs[4],rs[5],rs[6],rs[7],rs[8],rs[9],rs[10],'A')
            cursor.close()
            self.db.close()
        except pyodbc.Error as e:
            print(e)
        return c
    def actualizar(self,cliente):
        sql = "update Ventas.Clientes set nombre=(?),razonSocial=(?),limiteCredito=(?),direccion=(?),codigoPostal=(?), " \
              "rfc=(?),telefono=(?),email=(?),tipo=(?),idCiudad=(?) where idCliente=(?)"
        try:
            cursor = self.db.cursor()
            Values = [cliente.nombre,cliente.razonSocial,cliente.limiteCredito,cliente.direccion,
                      cliente.codigoPostal,cliente.rfc,cliente.telefono,cliente.email,cliente.tipo,cliente.idCiudad,cliente.idCliente]
            cursor.execute(sql,Values)
            self.db.commit()
            cursor.close()
            self.db.close()
        except pyodbc.Error as e:
            print(e)
    def eliminar(self,id):
        sql="update Ventas.Clientes set estatus='I' where idCliente=(?)"
        try:
            cursor = self.db.cursor()
            Values = [id]
            cursor.execute(sql, Values)
            self.db.commit()
            cursor.close()
            self.db.close()
        except pyodbc.Error as e:
            print(e)


from mynode import MyNode

class MyHashTable:
    INITIAL_CAPACITY = 5
    nodes = [None] * INITIAL_CAPACITY

    def add(self, data):
        index = hash(data) % len(self.nodes)

        if self.nodes[index] is None:
            print('has no values')
            self.nodes[index] = MyNode(data)
        else:
            print('adding value')
            self.nodes[index].add(MyNode(data))

    def lenght(self):
        size = 0

        for node in self.nodes:
            while node is not None:
                node = node.next
                size+=1

        return size


#!/usr/bin/env python
"""
 Copyright (c) 2018 Intel Corporation

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""

from __future__ import print_function
import sys
import os
from argparse import ArgumentParser
import cv2
import time
import logging as log
from openvino.inference_engine import IENetwork, IEPlugin

def build_argparser():
    parser = ArgumentParser()
    parser.add_argument("-m", "--model", help="Path to an .xml file with a trained model.", required=True, type=str)
    parser.add_argument("-i", "--input",
                        help="Path to video file or image. 'cam' for capturing video stream from camera", required=True,
                        type=str)
    parser.add_argument("-l", "--cpu_extension",
                        help="MKLDNN (CPU)-targeted custom layers.Absolute path to a shared library with the kernels "
                             "impl.", type=str, default=None)
    parser.add_argument("-pp", "--plugin_dir", help="Path to a plugin folder", type=str, default=None)
    parser.add_argument("-d", "--device",
                        help="Specify the target device to infer on; CPU, GPU, FPGA or MYRIAD is acceptable. Demo "
                             "will look for a suitable plugin for device specified (CPU by default)", default="CPU",
                        type=str)
    parser.add_argument("--labels", help="Labels mapping file", default=None, type=str)
    parser.add_argument("-pt", "--prob_threshold", help="Probability threshold for detections filtering",
                        default=0.5, type=float)

    return parser

def main():
    log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout)
    args = build_argparser().parse_args()
    model_xml = args.model
    model_bin = os.path.splitext(model_xml)[0] + ".bin"
    # Plugin initialization for specified device and load extensions library if specified
    log.info("Initializing plugin for {} device...".format(args.device))
    plugin = IEPlugin(device=args.device, plugin_dirs=args.plugin_dir)
    if args.cpu_extension and 'CPU' in args.device:
        plugin.add_cpu_extension(args.cpu_extension)

    # Read IR
    log.info("Reading IR...")
    net = IENetwork.from_ir(model=model_xml, weights=model_bin)

    if "CPU" in plugin.device:
        supported_layers = plugin.get_supported_layers(net)
        not_supported_layers = [l for l in net.layers.keys() if l not in supported_layers]
        if len(not_supported_layers) != 0:
            log.error("Following layers are not supported by the plugin for specified device {}:\n {}".
                      format(plugin.device, ', '.join(not_supported_layers)))
            log.error("Please try to specify cpu extensions library path in demo's command line parameters using -l "
                      "or --cpu_extension command line argument")
            sys.exit(1)
    assert len(net.inputs.keys()) == 1, "Demo supports only single input topologies"
    assert len(net.outputs) == 1, "Demo supports only single output topologies"
    input_blob = next(iter(net.inputs))
    out_blob = next(iter(net.outputs))
    log.info("Loading IR to the plugin...")
    exec_net = plugin.load(network=net, num_requests=2)
    # Read and pre-process input image
    n, c, h, w = net.inputs[input_blob]
    del net
    if args.input == 'cam':
        input_stream = 0
    else:
        input_stream = args.input
        assert os.path.isfile(args.input), "Specified input file doesn't exist"
    if args.labels:
        with open(args.labels, 'r') as f:
            labels_map = [x.strip() for x in f]
    else:
        labels_map = None

    cap = cv2.VideoCapture(input_stream)

    cur_request_id = 0
    next_request_id = 1

    log.info("Starting inference in async mode...")
    log.info("To switch between sync and async modes press Tab button")
    log.info("To stop the demo execution press Esc button")
    is_async_mode = True
    render_time = 0
    while cap.isOpened():
        ret, frame = cap.read()
        if not ret:
            break
        initial_w = cap.get(3)
        initial_h = cap.get(4)
        in_frame = cv2.resize(frame, (w, h))
        in_frame = in_frame.transpose((2, 0, 1))  # Change data layout from HWC to CHW
        in_frame = in_frame.reshape((n, c, h, w))

        # Main sync point:
        # in the truly Async mode we start the NEXT infer request, while waiting for the CURRENT to complete
        # in the regular mode we start the CURRENT request and immediately wait for it's completion
        inf_start = time.time()
        if is_async_mode:
            exec_net.start_async(request_id=next_request_id, inputs={input_blob: in_frame})
        else:
            exec_net.start_async(request_id=cur_request_id, inputs={input_blob: in_frame})
        if exec_net.requests[cur_request_id].wait(-1) == 0:
            inf_end = time.time()
            det_time = inf_end - inf_start

            # Parse detection results of the current request
            res = exec_net.requests[cur_request_id].outputs[out_blob]
            for obj in res[0][0]:
                # Draw only objects when probability more than specified threshold
                if obj[2] > args.prob_threshold:
                    xmin = int(obj[3] * initial_w)
                    ymin = int(obj[4] * initial_h)
                    xmax = int(obj[5] * initial_w)
                    ymax = int(obj[6] * initial_h)
                    class_id = int(obj[1])
                    # Draw box and label\class_id
                    color = (min(class_id * 12.5, 255), min(class_id * 7, 255), min(class_id * 5, 255))
                    cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), color, 2)
                    det_label = labels_map[class_id] if labels_map else str(class_id)
                    cv2.putText(frame, det_label + ' ' + str(round(obj[2] * 100, 1)) + ' %', (xmin, ymin - 7),
                                cv2.FONT_HERSHEY_COMPLEX, 0.6, color, 1)

            # Draw performance stats
            inf_time_message = "Inference time: N\A for async mode" if is_async_mode else \
                "Inference time: {:.3f} ms".format(det_time * 1000)
            render_time_message = "OpenCV rendering time: {:.3f} ms".format(render_time * 1000)
            async_mode_message = "Async mode is on. Processing request {}".format(cur_request_id) if is_async_mode else \
                "Async mode is off. Processing request {}".format(cur_request_id)

            cv2.putText(frame, inf_time_message, (15, 15), cv2.FONT_HERSHEY_COMPLEX, 0.5, (200, 10, 10), 1)
            cv2.putText(frame, render_time_message, (15, 30), cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1)
            cv2.putText(frame, async_mode_message, (10, int(initial_h - 20)), cv2.FONT_HERSHEY_COMPLEX, 0.5,
                        (10, 10, 200), 1)

        #
        render_start = time.time()
        cv2.imshow("Detection Results", frame)
        render_end = time.time()
        render_time = render_end - render_start

        key = cv2.waitKey(1)
        if key == 27:
            break
        if (9 == key):
            is_async_mode = not is_async_mode
            log.info("Switched to {} mode".format("async" if is_async_mode else "sync"))

        if is_async_mode:
            cur_request_id, next_request_id = next_request_id, cur_request_id

    cv2.destroyAllWindows()
    del exec_net
    del plugin

if __name__ == '__main__':
    sys.exit(main() or 0)


import re
import json
import logging
import pymongo
from scrapy import Request
from scrapy.spiders import Spider
from house.items import HouseItem
from scrapy_splash import SplashRequest
from house.db import Db

class HouseSpider(Spider):

    name = 'HouseSpider'
    lastId = 0
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36',
    }
    # 当前area
    curArea = {}

    def start_requests(self):
        area_coll = Db.getColl("areas")
        areas = area_coll.find().limit(1)
        for area in areas:
            HouseSpider.curArea =area
            area_key = area["key"]
            url = 'https://www.domain.com.au/sale/' + area_key + '/?sort=dateupdated-desc'
            yield Request(url, headers = self.headers)

    def parse(self, response):
        houses = response.css('.listing-result__address')
        for house in houses:
            url = house.css('::attr(href)').extract_first()
            # 房源详情页
            yield Request(url, callback=self.parseDetail,
                headers=self.headers)
        # 获取下一页
        link_btns = response.css('.paginator .is-outline')
        next_href = link_btns[len(link_btns) - 1].css("::attr(href)").extract_first()
        if next_href:
            next_href = 'https://www.domain.com.au' + next_href
            print(next_href)
            yield Request(next_href, callback=self.parse, headers = self.headers)
    def parseDetail(self, response):
        item = HouseItem()
        dataStr = response.text.split('window.renderizrData["page"] =')[1].split(';</script>')[0]
        data = json.loads(dataStr)
        item["address"] = data.get("address", '')
        item["office_banner"] = data.get("agencyBanner", '')
        item["office_logo"] = data.get("agencyLogo", '')
        item["office_name"] = data.get("agencyName", '')
        item["office_url"] = data.get("agencyProfileUrl", '')
        item["branding_color"] = data.get("brandingColor", '')

        item["agents"] = data.get("agents", '')

        item["created_on"] = data.get("createdOn", '')
        item["description"] = data.get("description", [])
        item["domain_says"] = data.get("domainSays", '')
        item["enable_child_listing_experiment"] = data.get("enableChildListingExperiment", '')
        item["enable_echoice"] = data.get("enableEchoice", '')
        item["enable_header_v2test"] = data.get("enableHeaderV2Test", '')
        item["enable_homepass"] = data.get("enableHomepass", '')
        item["enable_property_features"] = data.get("enablePropertyFeatures", '')
        item["gallery"] = data["gallery"]["slides"]
        item["header"] = data.get("header", '')
        item["headline"] = data.get("headline", '')
        item["house_id"] = data.get("id", '')
        item["inspection"] = data.get("inspection", '')
        item["is_archived"] = data.get("isArchived", '')
        item["is_land_estate"] = data.get("isLandEstate", '')
        item["listing_summary"] = data.get("listingSummary", '')
        item["listing_url"] = data.get("listingUrl", '')
        item["type"] = data.get("listingType", '')
        item["lat"] = data["map"]["latitude"]
        item["lng"] = data["map"]["longitude"]
        item["phone"] = data.get("phone", '')
        item["postcode"] = data.get("postcode", '')
        item["property_type"] = data.get("propertyType", '')
        item["schools"] = data["schoolCatchment"]["schools"]
        item["state_abbreviation"] = data.get("stateAbbreviation", '') #地区缩写
        item["street"] = data.get("street", '')
        item["street_number"] = data.get("streetNumber", '')
        item["suburb"] = data.get("suburb", '')
        item["suburb_insights"] = data.get("suburbInsights", '')
        item["what_isnearby"] = data.get("whatIsNearby", '')
        item["unit_number"] = data.get("unitNumber", '')

        # 自定义数据
        item["refId"] = data.get("id", "")
        item["region"] = HouseSpider.curArea.get("region", "")
        item["region_cn"] = HouseSpider.curArea.get("region_cn", "")
        item["region_id"] = HouseSpider.curArea.get("region_id", "")
        item["region_key"] = HouseSpider.curArea.get("region_key", "")

        item["district"] = HouseSpider.curArea.get("district", "")
        item["district_cn"] = HouseSpider.curArea.get("district_cn", "")
        item["district_id"] = HouseSpider.curArea.get("district_id", "")

        item["area"] = HouseSpider.curArea.get("name", "")
        item["area_cn"] = HouseSpider.curArea.get("cn", "")
        item["area_id"] = HouseSpider.curArea.get("id", "")

        item["property_type_id"] = self.getProperyType(data.get("propertyType", ''))
        item["price"] = data["listingSummary"].get("price", "")
        item["price_val"] = 0
        # 如果price有值
        price = item["price"]

        if "$" in price:
            # price = price.replace('*', '').replace("'", '').replace("+", "").replace("s","")
            # price = price.split('-')[0]
            # price = price.split(' ')[0]
            # a = price[price.find("$") + 1:].split(',')
            # c = ''.join(a)
            # 替换字符串
            price = price.split('-')[0]
            price =  re.sub("\D", "", price)
            item["price_val"] = price

        item["bed"] = data["listingSummary"].get("beds", "")
        item["bath"] = data["listingSummary"].get("baths", "")
        item["car"] = data["listingSummary"].get("parking", "")
        item["status"] = data["listingSummary"].get("status", "")

        # to do landarea & internal area
        return item

    # 获取房源类型id
    def getProperyType(self, key):
        typeDic = {
            "Duplex": 1,
            "House": 1,
            "Semi-Detached": 1,
            "Terrace": 1,
            "Villa": 1,
            "New House & Land": 1,
            "New Home Designs": 1,
            "Block of Units": 2,
            "Penthouse": 2,
            "Studio": 2,
            "Apartment / Unit / Flat": 2,
            "New Apartments / Off the Plan": 2,
            "Townhouse": 3,
            "Development Site": 4,
            "New land": 4,
            "Vacant land": 4,
            "Acreage / Semi-Rural": 5,
            "Farm": 5,
            "Rural": 5
        }
        return typeDic.get(key, 0)


#!/usr/bin/env python

import glob
import os
import pytest

import astrodata
import gemini_instruments

F2_DESCRIPTORS_TYPES = [
    ('detector_x_offset', float),
    ('detector_y_offset', float),
    ('nonlinearity_coeffs', list),
    ('pixel_scale', float),
]

@pytest.fixture
def f2_files(path_to_inputs):
    def get_files(instrument):
        return glob.glob(os.path.join(path_to_inputs, instrument, "*fits"))

    gemini_files = []
    gemini_files.extend(get_files("F2"))
    gemini_files.sort()

    yield gemini_files

@pytest.mark.parametrize("descriptor,expected_type", F2_DESCRIPTORS_TYPES)
def test_descriptor_matches_type(descriptor, expected_type, f2_files):
    for _file in f2_files:
        ad = astrodata.open(_file)

        value = getattr(ad, descriptor)()

        assert isinstance(value, expected_type) or value is None, \
            "Assertion failed for file: {}".format(_file)

if __name__ == '__main__':
    pytest.main()


from basic_units import radians, degrees, cos
from pylab import figure, show, nx

x = nx.arange(0, 15, 0.01) * radians

fig = figure()
fig.subplots_adjust(hspace=0.3)
ax = fig.add_subplot(211)
ax.plot(x, cos(x), xunits=radians)

ax = fig.add_subplot(212)
ax.plot(x, cos(x), xunits=degrees)

show()


import boto3
import configparser,os
import logging
from botocore.exceptions import ClientError

config = configparser.ConfigParser()
AWS_PROFILE = "default"
config.read(os.path.expanduser("~/.aws/credentials"))
access_id = config.get(AWS_PROFILE, "aws_access_key_id")
access_key = config.get(AWS_PROFILE, "aws_secret_access_key")
# Create SQS client
sqs = boto3.client('sqs',
                    aws_access_key_id = access_id,
                    aws_secret_access_key = access_key
)

queue_url = 'https://sqs.eu-west-1.amazonaws.com/094529590173/streaming-app_raj_raw'

def send_sqs_message(sqs_queue_url, msg_body):
    """

    :param sqs_queue_url: String URL of existing SQS queue
    :param msg_body: String message body
    :return: Dictionary containing information about the sent message. If
        error, returns None.
    """

    # Send the SQS message
    sqs_client = boto3.client('sqs')
    try:
        msg = sqs_client.send_message(QueueUrl=sqs_queue_url,
                                      MessageBody=msg_body)
    except ClientError as e:
        logging.error(e)
        return None
    return msg

def main():
    """Exercise send_sqs_message()"""

    # Assign this value before running the program
    sqs_queue_url = queue_url

    # Set up logging
    logging.basicConfig(level=logging.INFO,
                        format='%(levelname)s: %(asctime)s: %(message)s')

    # Send some SQS messages
    for i in range(1, 6):
        msg_body = f'{{"row" : {i} , "name" : "raj", "age" : 34}}'
        msg = send_sqs_message(sqs_queue_url, msg_body)
        if msg is not None:
            logging.info(f'Sent SQS message ID: {msg["MessageId"]}')

if __name__ == '__main__':
    main()


from telegram import ReplyKeyboardMarkup, KeyboardButton
from clarifai.rest import ClarifaiApp

import settings
import pprint

def get_keyboard():
    contact_button = KeyboardButton('Прислать контакты', request_contact=True)
    location_button = KeyboardButton('Прислать координаты', request_location=True)

    my_keyboard = ReplyKeyboardMarkup([['/start', '/stop', '/sith'],
                                       [contact_button, location_button, 'Сменить аватар'],
                                      ['Заполнить анкету', 'Покозать inline-клавиатуру']])
    return my_keyboard

def is_sword(filename):
    image_has_sword = False
    app = ClarifaiApp(api_key=settings.CLARIFAI_API_KEY)
    model = app.public_models.general_model
    response = model.predict_by_filename(filename, max_concepts=5)

    pp =  pprint.PrettyPrinter(indent=4)
    pp.pprint(response)
    if response['status']['code'] == 10000:
        for concept in response['outputs'][0]['data']['concepts']:
            if concept['name'] == 'sword':
                print("Sword is here!")
                image_has_sword = True
    return image_has_sword

if __name__=="__main__":
    is_sword('sith/undeaddu2.jpg')


#!/usr/bin/env python3

import sys, re

if __name__ == "__main__":
    for line in sys.stdin:
        line = line.strip()
        if line.startswith('>'):
            new_header = []
            m_seq = re.search(r"^>(?P<seq_id>\S+).+", str(line))
            if m_seq is not None:
                new_header.append(str(m_seq.group('seq_id')))
            m_size = re.search(r"(?P<size>size=\S+);", str(line))
            if m_size is not None:
                new_header.append(str(m_size.group('size')))
            m_count = re.search(r"(?P<count>count=\S+);", str(line))
            if m_count is not None:
                new_header.append(str(m_count.group('count')))
            m_sample = re.search(r"(?P<sample>merged_sample=\{.+\});",str(line))
            if m_sample is not None:
                new_header.append(str(m_sample.group('sample')).replace(" ", ""))
            print(">" + ";".join(new_header))
        else:
            print(line)


import cv2
import os

def my_main_function():
	while(True):
		cap = cv2.VideoCapture('rtsp://192.168.0.143:8554/unicast')
		#frameRate = cap.get(5) #frame rate
		frameRate = 25
		x=1
		savepath = 'cframe'
		while(cap.isOpened()):
			frameId = cap.get(1) #current frame number
			ret, frame = cap.read()
			if (ret != True):
		  		break
			if (frameId % frameRate == 0):
				os.chdir(savepath)
				cv2.imwrite('cframe'+'.jpg', frame)
				os.chdir('..')
				print("image",x)
				x=x+1
		cap.release()
		print ("Done!")

if __name__=='__main__':
	try:
		my_main_function()
	except:
		my_main_function()
	else:
		my_main_function()


import xmltodict
import pytest

from jmeter_api.assertions.jsr223.elements import JSR223Assertion, ScriptLanguage
from jmeter_api.basics.utils import tag_wrapper

class TestJSR223AssertionRender:
    def test_script_language(self):
        element = JSR223Assertion(script_language=ScriptLanguage.JAVA)
        rendered_doc = element.to_xml()
        parsed_doc = xmltodict.parse(tag_wrapper(rendered_doc,'test_result'))
        for tag in parsed_doc['test_result']['JSR223Assertion']['stringProp']:
            if tag['@name'] == 'scriptLanguage':
                assert tag['#text'] == 'java'

    def test_cache_key(self):
        element = JSR223Assertion(cache_key=False)
        rendered_doc = element.to_xml()
        parsed_doc = xmltodict.parse(tag_wrapper(rendered_doc,'test_result'))
        for tag in parsed_doc['test_result']['JSR223Assertion']['stringProp']:
            if tag['@name'] == 'cacheKey':
                assert tag['#text'] == 'false'

    def test_filename(self):
        element = JSR223Assertion(filename="./jmeter_api/basics/jsr223_test.groovy")
        rendered_doc = element.to_xml()
        parsed_doc = xmltodict.parse(tag_wrapper(rendered_doc,'test_result'))
        for tag in parsed_doc['test_result']['JSR223Assertion']['stringProp']:
            if tag['@name'] == 'filename':
                assert tag['#text'] == "./jmeter_api/basics/jsr223_test.groovy"

    def test_script(self):
        sc = """var a=2
vars.put("some value",a)
log("value added")"""
        element = JSR223Assertion(script=sc)
        rendered_doc = element.to_xml()
        parsed_doc = xmltodict.parse(tag_wrapper(rendered_doc,'test_result'))
        for tag in parsed_doc['test_result']['JSR223Assertion']['stringProp']:
            if tag['@name'] == 'script':
                assert tag['#text'] == sc

    def test_hashtree_contain(self):
        element = JSR223Assertion()
        rendered_doc = element.to_xml()
        assert '<hashTree />' in rendered_doc


import cv2
import numpy as np
from matplotlib import pyplot as plt

cap = cv2.VideoCapture(1) # number 0 for one camera
#fourcc = cv2.VideoWriter_fourcc(*'XVID')
#out  = cv2.VideoWriter('output.avi',fourcc,24.0,(640,480)) #params 3 = fram rate speed fram for save

# GRAY SCALE FOR ALL PIC
while(True): #video is pics
    ret,fram = cap.read()
    #gray = cv2.cvtColor(fram,cv2.COLOR_BGR2GRAY) # cv2 =>blue - green -red
   # cv2.line(fram,(100,200)(200,400),(255,0,0),5)
    #out.write(fram)
    cv2.imshow('cameras',fram)
    if cv2.waitKey(1) & 0XFF == ord('q'):
        break

cap.release()
#out.release()
cv2.destroyAllWindows()


import numpy as np

# Creating the convolutional layer Class
class Convolution_Layer:

    def __init__(self, inputs_channel, num_filters, kernel_size, padding, stride, learning_rate, name):
        # weight size: (F, C, K, K)
        # bias size: (F)
        self.Filters = num_filters
        self.Kernels = kernel_size
        self.Channels = inputs_channel

        self.weights = np.zeros((self.Filters, self.Channels, self.Kernels, self.Kernels))
        self.bias = np.zeros((self.Filters, 1))
        # Weights initialization
        for i in range(0,self.Filters):
            self.weights[i,:,:,:] = np.random.normal(loc=0, scale=np.sqrt(1./(self.Channels*self.Kernels*self.Kernels)), size=(self.Channels, self.Kernels, self.Kernels))

        self.pad = padding
        self.ST = stride
        self.LRate = learning_rate
        self.Layer_name = name

    # Zero padding function
    def zero_pad(self, inputs, size):
        wid, hei = inputs.shape[0], inputs.shape[1]
        new_wid = 2 * size + wid
        new_hei = 2 * size + hei
        out = np.zeros((new_wid, new_hei))
        out[size:wid+size, size:hei+size] = inputs
        return out
    # Forward Propagation function
    def forward_propagation(self, inputs):
        """
        this forward is doing the convolution and storing the results into the feature maps
        (Calculating the feature maps)
        """
        # input size: (C, W, H)
        # output size: (N, F ,WW, HH)
        Channels = inputs.shape[2]
        Width = inputs.shape[0]+2*self.pad
        Height = inputs.shape[1]+2*self.pad
        self.inputs = np.zeros(( Width, Height, Channels))
        for c in range(inputs.shape[2]):
            self.inputs[:,:, c] = self.zero_pad(inputs[:,:, c], self.pad)
        WW = int((Width - self.Kernels)/self.ST) + 1
        HH = int((Height - self.Kernels)/self.ST) + 1
        # print("width and height are:", WW, HH)
        feature_maps = np.zeros(( WW, HH, self.Filters))
        # print("the shape of the inputs is",self.inputs.shape)
        # print("the shape of the weights is" , self.weights.shape)

        # Doing the Convolution
        try:
            for f in range(self.Filters):
                for ch in range(Channels):
                    for w in range(WW):
                        for h in range(HH):
                            feature_maps[w,h,f]=np.sum(self.inputs[w:w + self.Kernels, h:h + self.Kernels, ch]*self.weights[f, ch, :, :])+self.bias[f]

                        #print(feature_maps[w,h,f])
        except IndexError:
            print("I m having Index error")
        # print("the feauture map shape is:",feature_maps.shape )
        return feature_maps

    # Forward Propagation function
    def backward_propagation(self, dy):

        Width, Height, Channels = self.inputs.shape
        dx = np.zeros(self.inputs.shape)
        dw = np.zeros(self.weights.shape)
        db = np.zeros(self.bias.shape)

        Width, Height, F = dy.shape
        for f in range(F):
            for ch in range(Channels):
                for w in range(Width):
                    for h in range(Height):
                        dw[f,ch,:,:]+=dy[w,h,f]*self.inputs[w:w+self.Kernels,h:h+self.Kernels, ch]
                        dx[w:w+self.Kernels,h:h+self.Kernels, ch]+=dy[w,h,f]*self.weights[f,ch,:,:]

        # print("the shapes for dw, dx, dy are: ",dw.shape, dx.shape, dy.shape)
        for f in range(F):
            db[f] = np.sum(dy[:, :, f])

        self.weights -= self.LRate * dw
        self.bias -= self.LRate * db
#         print("the weights after backptop: ", self.weights)
        return dx

    # Function for extracting weights and biases
    def extract_weights(self):
        return {self.Layer_name +'.weights':self.weights, self.Layer_name +'.bias':self.bias}

    # Function for feeding or filling weights and biases
    def feed_weights(self, weights, bias):
        self.weights = weights
        self.bias = bias

# Creating the MAxpooling layer Class
class Maxpool_Layer:

    def __init__(self, pool_size, stride, name):
        self.pool = pool_size
        self.ST = stride
        self.Layer_name = name
    # Forward Propagation function
    def forward_propagation(self, inputs):
        try:
            # print("In the forward pass of maxpooling")
            self.inputs = inputs
            Width, Height, Channels = inputs.shape
            # print("the object types are", type(Channels),type(W),type(H))
            new_width = int((Width - self.pool)/self.ST)+ 1
            new_height = int((Height - self.pool)/self.ST) + 1
            # print("the new width",new_width,new_height)

            l1 = int(Width/self.ST)
            l2 = int(Height/self.ST)

            # print("the range are ",l1,l2, C)
            out = np.zeros(( new_width, new_height, Channels))
            # print("In maxpooling, the value of C,W,H,new_width,new_height", C,W,H,new_width,new_height)
            for c in range(Channels):
                for w in range(l1):
                    for h in range(l2):
                        #print("the calculated value is",np.max(self.inputs[c, w*self.ST:w*self.ST+self.pool, h*self.ST:h*self.ST+self.pool]))
                        out[ w, h, c] = np.max(self.inputs[ w*self.ST:w*self.ST+self.pool, h*self.ST:h*self.ST+self.pool, c])
                        #print("the out value is",out[c, w, h])
            # print("the shape after maxpool:",out.shape)
        except:
            print("Error in Maxpooling Layer")
        # print("the shape of max pool:", out.shape)
        return out

    # Backward Propagation function
    def backward_propagation(self, dy):
        masking = np.ones_like(self.inputs)*0.25
        return masking*(np.repeat(np.repeat(dy[0:-4, 0:-4 :],2,axis=0),2,axis=1))

    # Function for extracting weights and biases
    def extract_weights(self):
        return

# Creating the Fully Connected layer Class
class FullyConnected_Layer:

    def __init__(self, num_inputs, num_outputs, learning_rate, name):
        self.weights = 0.01*np.random.rand(num_inputs, num_outputs)
        self.bias = np.zeros((num_outputs, 1))
        self.LRate = learning_rate
        self.Layer_name = name

    # Forward Propagation function
    def forward_propagation(self, inputs):
        self.inputs = inputs
        return np.dot(self.inputs, self.weights) + self.bias.T

    # Backward Propagation function
    def backward_propagation(self, dy):

        if dy.shape[0] == self.inputs.shape[0]:
            dy = dy.T
        dw = dy.dot(self.inputs)
        db = np.sum(dy, axis=1, keepdims=True)
        dx = np.dot(dy.T, self.weights.T)

        self.weights -= self.LRate * dw.T
        self.bias -= self.LRate * db

        return dx

    # Function for extracting weights and biases
    def extract_weights(self):
        return {self.Layer_name +'.weights':self.weights, self.Layer_name +'.bias':self.bias}

    # Function for feeding or filling weights and biases
    def feed_weights(self, weights, bias):
        self.weights = weights
        self.bias = bias

# Creating the Flattening layer Class
class Flattening_Layer:
    def __init__(self):
        pass

    # Forward Propagation function
    def forward_propagation(self, inputs):
        self.Width, self.Height, self.Channels, = inputs.shape
        return inputs.reshape(1, self.Channels*self.Width*self.Height)

    # Backward Propagation function
    def backward_propagation(self, dy):
        return dy.reshape( self.Width, self.Height, self.Channels)

    # Function for extracting weights and biases
    def extract_weights(self):
        return

# Creating the ReLu Activation Class
class ReLu_Activation:
    def __init__(self):
        pass

    # Forward Propagation function
    def forward_propagation(self, inputs):
        self.inputs = inputs
        relu = inputs.copy()
        relu[relu < 0] = 0
        return relu

    # Backward Propagation function
    def backward_propagation(self, dy):
        dx = dy.copy()
        dx[self.inputs < 0] = 0
        return dx

    # Function for extracting weights and biases
    def extract_weights(self):
        return

# Creating the Softmax Layer Class
class Softmax_Layer:
    def __init__(self):
        pass
    # Forward Propagation function
    def forward_propagation(self, inputs):
        exp_prob = np.exp(inputs, dtype=np.float)
        self.out = exp_prob/np.sum(exp_prob)
        # print("Completing the forward pass of softmax", self.out.shape)
        # print("Completing the softmax with values", np.unique(self.out).shape)
        return self.out

    # Backward Propagation function
    def backward_propagation(self, dy):
        return self.out.T - dy.reshape(dy.shape[1],1)

    # Function for extracting weights and biases
    def extract_weights(self):
        return


from qtpy.QtWidgets import QWidget, QHBoxLayout, QLabel, QSizePolicy, QToolButton, QVBoxLayout, QDialog
from qtpy.QtCore import Qt, QEvent, QMetaObject, QSize
from qtpy.QtGui import QIcon, QPixmap

from .._borderless.win32 import BorderlessWindow
from .._utils import MINIMIZE_ICON, CLOSE_ICON, MAXIMIZE_ICON, RESTORE_ICON
from .. import _resources_rc

_style = """
QToolButton {{
  background-color: transparent;
  border: transparent;
  padding: 0 10px;
  height: 20px;
}}

QToolButton:hover {{
  background-color: {};
  border: transparent;
}}

#btnClose:hover {{
  background-color: #d11919;
  border: transparent;
}}

"""

class _TitleBar(QWidget):
    def __init__(self, parent=None):
        super(_TitleBar, self).__init__(parent)
        self.h_layout_content = QHBoxLayout()
        self.title_layout_content = QHBoxLayout()
        self.title_layout_content.setSpacing(5)
        self.title_layout_content.setContentsMargins(5, 0, 0, 0)
        self.h_layout_content.setSpacing(0)
        self.h_layout_content.setContentsMargins(0, 0, 0, 0)
        self.setLayout(self.h_layout_content)

        self.application_icon = QToolButton()
        self.application_icon.setObjectName('applicationLogo')

        self.lbl_title = QLabel('Title')
        self.lbl_title.setObjectName('lblTitle')
        self.lbl_title.setAlignment(Qt.AlignLeft | Qt.AlignVCenter)
        self.lbl_title.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed)

        sp_buttons = QSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)

        self.btn_minimize = QToolButton()
        self.btn_minimize.setObjectName('btnMinimize')
        self.btn_minimize.setSizePolicy(sp_buttons)

        self.btn_restore = QToolButton()
        self.btn_restore.setObjectName('btnRestore')
        self.btn_restore.setSizePolicy(sp_buttons)
        self.btn_restore.setVisible(False)

        self.btn_maximize = QToolButton()
        self.btn_maximize.setObjectName('btnMaximize')
        self.btn_maximize.setSizePolicy(sp_buttons)

        self.btn_close = QToolButton()
        self.btn_close.setObjectName('btnClose')
        self.btn_close.setSizePolicy(sp_buttons)

        color = '#%02x%02x%02x' % self.palette().midlight().color().getRgb()[:-1]
        button_style = _style.format(color)

        self.btn_minimize.setIcon(QIcon(QPixmap(MINIMIZE_ICON)))
        self.btn_restore.setIcon(QIcon(QPixmap(RESTORE_ICON)))
        self.btn_maximize.setIcon(QIcon(QPixmap(MAXIMIZE_ICON)))
        self.btn_close.setIcon(QIcon(QPixmap(CLOSE_ICON)))
        self.btn_minimize.setStyleSheet(button_style)
        self.btn_restore.setStyleSheet(button_style)
        self.btn_maximize.setStyleSheet(button_style)
        self.btn_close.setStyleSheet(button_style)
        self.application_icon.setStyleSheet("QToolButton { background-color: transparent; border: transparent;}")

        self.title_layout_content.addWidget(self.application_icon)

        self.title_layout_content.addWidget(self.lbl_title)
        self.h_layout_content.addLayout(self.title_layout_content)
        self.h_layout_content.addWidget(self.btn_minimize)
        self.h_layout_content.addWidget(self.btn_restore)
        self.h_layout_content.addWidget(self.btn_maximize)
        self.h_layout_content.addWidget(self.btn_close)

class _WindowsTitleBar(_TitleBar):
    def __init__(self, window, parent=None):
        super().__init__(parent)
        self._window = window
        self._window.installEventFilter(self)

        self.setObjectName('titleBar')
        self.setAttribute(Qt.WA_StyledBackground, True)

        icon = self._window.windowIcon() if not self._window.windowIcon().isNull() else self._window._window.windowIcon()
        self.application_icon.setIcon(icon)
        self.application_icon.setIconSize(QSize(12, 12))
        self._window.windowIconChanged.connect(self.on_window_icon_changed)
        self._window._window.windowIconChanged.connect(self.on_window_icon_changed)

        title = self._window.windowTitle() if self._window.windowTitle() else self._window._window.windowTitle()
        self.lbl_title.setText(title)
        self._window.windowTitleChanged.connect(self.on_window_title_changed)
        self._window._window.windowTitleChanged.connect(self.on_window_title_changed)

        self.btn_minimize.clicked.connect(self.on_btn_minimize_clicked)
        self.btn_restore.clicked.connect(self.on_btn_restore_clicked)
        self.btn_maximize.clicked.connect(self.on_btn_maximize_clicked)
        self.btn_close.clicked.connect(self.on_btn_close_clicked)

        QMetaObject.connectSlotsByName(self)

    def eventFilter(self, obj, event):
        if event.type() == QEvent.WindowStateChange:
            if self._window.windowState() == Qt.WindowMaximized:
                self.btn_maximize.setVisible(False)
                self.btn_restore.setVisible(True)
            else:
                self.btn_maximize.setVisible(True)
                self.btn_restore.setVisible(False)

            self.btn_restore.setAttribute(Qt.WA_UnderMouse, False)
            self.btn_maximize.setAttribute(Qt.WA_UnderMouse, False)

        return super().eventFilter(obj, event)

    def on_btn_minimize_clicked(self):
        self._window.setWindowState(Qt.WindowMinimized)

    def on_btn_restore_clicked(self):
        self._window.setWindowState(Qt.WindowNoState)
        self.btn_maximize.setVisible(True)
        self.btn_restore.setVisible(False)

    def on_btn_maximize_clicked(self):
        self._window.setWindowState(Qt.WindowMaximized)
        self.btn_maximize.setVisible(False)
        self.btn_restore.setVisible(True)

    def on_btn_close_clicked(self):
        self._window.close()

    def on_window_title_changed(self, title):
        self.lbl_title.setText(title)

    def on_window_icon_changed(self, icon):
        self.application_icon.setIcon(icon)

class ModernWindow(BorderlessWindow):
    def __init__(self, window):
        self._window = window
        super().__init__()
        self.hLayout = QVBoxLayout(self)
        self.hLayout.setContentsMargins(0, 0, 0, 0)
        self.hLayout.setSpacing(0)

        self.title_bar = _WindowsTitleBar(self, self)

        self.add_window_mover(self.title_bar.lbl_title)
        self.hLayout.addWidget(self.title_bar)
        self.hLayout.addWidget(window)

    def closeEvent(self, event):
        self._window.closeEvent(event)
        super().closeEvent(event)

class ModernDialog(QDialog, BorderlessWindow):
    def __init__(self, parent):
        self._window = parent
        QDialog.__init__(self, parent=parent)
        BorderlessWindow.__init__(self, parent=self)

        self.window_content = QWidget(self)
        self.hLayout = QVBoxLayout(self)
        self.hLayout.setContentsMargins(0, 0, 0, 0)
        self.hLayout.setSpacing(0)

        self.title_bar = _WindowsTitleBar(self, self)
        self.title_bar.setAttribute(Qt.WA_StyledBackground, False)
        self.add_window_mover(self.title_bar.lbl_title)

        self.title_bar.application_icon.setVisible(False)
        self.title_bar.btn_close.setVisible(False)
        self.title_bar.btn_maximize.setVisible(False)
        self.title_bar.btn_minimize.setVisible(False)
        self.title_bar.btn_restore.setVisible(False)

        self.hLayout.addWidget(self.title_bar)
        self.hLayout.addWidget(self.window_content)

    def setLayout(self, layout):
        self.window_content.setLayout(layout)


import yaml

class ReadYaml(object):
    def __init__(self, file_path, encoding):
        with open(file_path, "r", encoding=encoding) as file:
            self.configMap = yaml.load(file, Loader=yaml.FullLoader)

    def get(self, key_path):
        key_arr = key_path.split(".")
        temp_map = self.configMap
        for index in range(len(key_arr)):
            if index != len(key_arr) - 1:
                temp_map = self.configMap[key_arr[index]]
            else:
                return temp_map[key_arr[index]]

    def write(self, data, path):
        with open(path, 'w', encoding='utf-8') as file:
            yaml.dump(data=data, stream=file, allow_unicode=True)

if __name__ == "__main__":
    res = ReadYaml("E:\\SubjectNetwork-Download\\resources\\application.yml", "utf-8")
    username = res.get("base-url")
    print(username)

    apiData = {"base-url": "http://www.zxxk.com11"}
    res.configMap.update(apiData)
    with open("E:\\SubjectNetwork-Download\\resources\\application.yml", 'w', encoding='utf-8') as f:
        yaml.dump(data=res.configMap, stream=f, allow_unicode=True)

# 获取当前脚本所在文件夹路径
# curPath = os.path.dirname(os.path.realpath(__file__))
# 获取yaml文件路径
# yamlPath = os.path.join(curPath, "cfgyaml.yaml")


#Import Splinter, BeautifulSoup, and Pandas
from splinter import Browser
from bs4 import BeautifulSoup as soup
import pandas as pd
#from webdriver_manager.chrome import ChromeDriverManager
import datetime as dt
import requests

def scrape_all():
# Set the executable path and initialize Splinter
#executable_path = {'executable_path': ChromeDriverManager().install()}
    browser = Browser('chrome', executable_path="chromedriver",headless=True)

    news_title, news_paragraph= mars_news(browser)
    hemisphere_image_urls= hemisphere(browser)
    # Run all scraping functions and store results in dictionary
    data={
        "news_title": news_title,
        "news_paragraph": news_paragraph,
        "featured_image": featured_image(browser),
        "facts": mars_facts(),
        "hemispheres": hemisphere_image_urls,
        "last_modified": dt.datetime.now()
    }

    # Stop webdriver and return data
    browser.quit()
    return data

def mars_news(browser):
    # Visit the mars nasa news site
    url = 'https://data-class-mars.s3.amazonaws.com/Mars/index.html'
    browser.visit(url)

    # Optional delay for loading the page
    browser.is_element_present_by_css('div.list_text', wait_time=1)

    # Convert the browser html to a soup object and then quit the browser
    html = browser.html
    news_soup = soup(html, 'html.parser')
    try:
        #slide_elem looks for <ul /> tags and descendents <li />
        # the period(.) is used for selecting classes such as item_list
        slide_elem= news_soup.select_one('ul.item_list li.slide')

        # Chained the (.find) to slide_elem which says this variable holds lots of info, so look inside to find this specific entity
        # Get Title
        news_title=slide_elem.find('div', class_= 'content_title').get_text()
        # Get article body
        news_p= slide_elem.find('div', class_='article_teaser_body').get_text()

    except AttributeError:
        return None,None

    return news_title, news_p

def featured_image(browser):
    # ### JPL Space Images Featured Image
    # Visit URL
    url = 'https://data-class-jpl-space.s3.amazonaws.com/JPL_Space/index.html'
    browser.visit(url)

    # Find and click the full image button
    full_image_elem = browser.find_by_tag('button')[1]
    full_image_elem.click()

    # Parse the resulting html with soup
    html = browser.html
    img_soup = soup(html, 'html.parser')
    #img_soup
    try:
        # find the relative image url
        img_url_rel = img_soup.find('img', class_='fancybox-image').get('src')
        #img_url_rel
    except AttributeError:
        return None
    # Use the base url to create an absolute url
    img_url = f'https://data-class-jpl-space.s3.amazonaws.com/JPL_Space/{img_url_rel}'
    return img_url

# ### Mars Facts
def mars_facts():
    try:
        df = pd.read_html('https://data-class-mars-facts.s3.amazonaws.com/Mars_Facts/index.html')[0]
        #df.head()
    except BaseException:
        return None
    df.columns=['Description', 'Mars', 'Earth']
    df.set_index('Description', inplace=True)
    df

    return df.to_html()

# # D1: Scrape High-Resolution Mars’ Hemisphere Images and Titles

# ### Hemispheres
def hemisphere(browser):
    # 1. Use browser to visit the URL
    url = 'https://data-class-mars-hemispheres.s3.amazonaws.com/Mars_Hemispheres/index.html'

    browser.visit(url)

    response = requests.get(url)
    test  = soup(response.text,'html.parser')
    #item1 = test.find_all('div', class_='item')

    #main_url  = 'https://data-class-mars-hemispheres.s3.amazonaws.com/Mars_Hemispheres/'
    #title = test.find('h3').text
    #part_img = test.find('a',class_='itemLink product-item')['href']
    #browser.visit(main_url+part_img)

    #print(main_url+part_img)

    # 2. Create a list to hold the images and titles.
    main_url = 'https://data-class-mars-hemispheres.s3.amazonaws.com/Mars_Hemispheres/'
    hemisphere_image_urls = []
    items = test.find_all('div', class_ ='item')
    # 3. Write code to retrieve the image urls and titles for each hemisphere.
    for item in items:
        #hemispheres = {}
        title = item.find('h3').text
        part_img_url = item.find('a', class_= 'itemLink product-item')['href']
        browser.visit(main_url+part_img_url)
        part_img_html = browser.html
        test = soup(part_img_html,'html.parser')
        img_url = main_url+test.find('img',class_='wide-image')['src']
        hemisphere_image_urls.append({'title':title,'img_url':img_url})

# 4. Print the list that holds the dictionary of each image url and title.
    return hemisphere_image_urls

# 5. Quit the browser
if __name__== "__main__":
    # If running as script, print scrapped data
    print(scrape_all())


# -*- coding: utf-8 -*-
import imp

def environment_tester():
    state = 0
    reason = 'Unknown reason'
    try:
        imp.find_module('win32api')
        state = 1
        reason = 'Windows python environment is ready'
    except ImportError:
        state = 0
        reason = 'Non-Windows system or windows support module not ready' + ImportError.message()
    return state,reason