#!/usr/bin/env python
# coding: utf-8

# # COEN 140 Final Project - Music Genre Classifer

# In[241]:

import os
import json
import numpy as np
import scipy
import pandas as pd
import librosa as lb
import warnings

from sklearn.model_selection import train_test_split
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis
from sklearn.linear_model import LogisticRegression
from sklearn.cluster import KMeans

# ## Import functions

# In[3]:

def import_features(n=None):
    return pd.read_csv('fma_metadata/features.csv', header=[0,1,2], index_col=0, nrows=n)

def import_tracks(n=None, col=':'):
    return pd.read_csv('fma_metadata/tracks.csv', header=[0,1], index_col=0, nrows=n)

def import_genres(n=None):
    return pd.read_csv('fma_metadata/genres.csv', header=0, index_col=0, nrows=n)

# ## Feature extraction function

# In[3]:

def get_song_features(name):
    # Get the file path to an included audio example
    file = os.path.join(os.getcwd(), "test_songs", name)

    # Load into waveform 'y', sampling rate 'sr'
    y, sr = lb.load(file)
    print('> \'{}\' successfully loaded'.format(name))

    ## Extract all features
    df = {}

    df["chroma_stft"] = lb.feature.chroma_stft(y=y, sr=sr)
    df["chroma_cqt"] = lb.feature.chroma_cqt(y=y, sr=sr)
    df["chroma_cens"] = lb.feature.chroma_cens(y=y, sr=sr)

    df["tonnetz"] = lb.feature.tonnetz(y=y, sr=sr)
    df["mfcc"] = lb.feature.mfcc(y=y, sr=sr)

    df["spectral_centroid"] = lb.feature.spectral_centroid(y=y, sr=sr)
    df["spectral_bandwidth"] = lb.feature.spectral_bandwidth(y=y, sr=sr)
    df["spectral_contrast"] = lb.feature.spectral_contrast(y=y, sr=sr)
    df["spectral_rolloff"] = lb.feature.spectral_rolloff(y=y, sr=sr)

    df["rmse"] = lb.feature.rms(y=y)
    df["zcr"] = lb.feature.zero_crossing_rate(y=y)

    print('> Successfully extracted into dict')
    return df

# In[4]:

## format new song features

# fetch new song features as dict
feat = get_song_features("one_summers_day.mp3")

# fetch empty array with correct format, then append empty row
df = import_features(0).append(pd.Series(dtype=float), ignore_index=True)

# apply stats to new song features
stats = ['kurtosis','max','mean','median','min','skew','std']
funcs = [scipy.stats.kurtosis, np.amax, np.mean, np.median, np.amin, scipy.stats.skew, np.std]

for ft in df.columns.unique(0):
    for st, fn in zip(stats, funcs):
        df.loc[:,(ft,st)] = fn(feat[ft], axis=1)

print('> Successfully applied statistics')

# ## Classifier Helper Functions

# In[4]:

def format_test_valid(X, y, drop_unique=False):
    # parse genres into array
    y = y.apply(lambda g: json.loads(g))

    # select/remove empty genres
    eg = y.index[y.map(lambda g: len(g)==0)]
    X = X.drop(eg)
    y = y.drop(eg)

    # split into train/validate groups
    Xt, Xv, yt, yv = train_test_split(X, y, test_size=0.2, random_state=1)

    # remove genres with only 1 entry
    if drop_unique:
        ug = yt.drop_duplicates(False).index
        Xt = Xt.drop(ug)
        yt = yt.drop(ug)

    return Xt, Xv, yt, yv

def format_track_data(data, cols=None):
    # select specified columns
    data = data.loc[:,('track',cols)]
    data.columns = cols

    # parse genres into array
    data = data.applymap(lambda t: json.loads(t))

    return data

def score(a, b):
    assert len(a) == len(b), 'Arrays are not the same size'

    c = 0
    for v1, v2 in zip(a,b):
        if isinstance(v2, (int, np.integer)):
            if (v1==v2):
                c=c+1
        else:
            if v1 in v2:
                c=c+1
    return c/len(a)

def print_scores(ts, vs, name='New'):
    print('> {} Scores:'.format(name))
    print('Training score: {:.3f}\nValidation score: {:.3f}\n'.format(ts, vs))

# ## Classifier Implementations
# ### LDA Function

# In[264]:

def m1_LDA(X, y, score_method='default', verbose=False):
    #split data
    Xt, Xv, yt, yv = train_test_split(X, y, test_size=0.2)
    yt1 = yt.apply(lambda g: g[0])

    # fitting
    lda = LinearDiscriminantAnalysis()
    lda.fit(Xt, yt1)

    # predictions
    pt = lda.predict(Xt)
    terr = score(pt,yt)

    pv = lda.predict(Xv)
    verr = score(pv,yv)

    if verbose:
        print_scores(terr, verr, 'LDA')

    return verr

# ### QDA Function

# In[227]:

def m2_QDA(X, y, score_method='default', verbose=False):
    #split data
    Xt, Xv, yt, yv = train_test_split(X, y, test_size=0.2)
    yt1 = yt.apply(lambda g: g[0])

    # drop unique tracks
    ug = yt1.drop_duplicates(False).index
    Xt = Xt.drop(ug)
    yt = yt.drop(ug)
    yt1 = yt1.drop(ug)

    # fitting
    qda = QuadraticDiscriminantAnalysis(tol=10**-10)
    qda.fit(Xt, yt1)

    # predictions
    pt = qda.predict(Xt)
    terr = score(pt,yt)

    pv = qda.predict(Xv)
    verr = score(pv,yv)

    if verbose:
        print_scores(terr, verr, 'QDA')

    return verr

# ### KMC Function

# In[198]:

def m3_KMC(X, y, init_method='k-means++', verbose=False):
    #split data
    Xt, Xv, yt, yv = train_test_split(X, y, test_size=0.2)
    yt1 = yt.apply(lambda g: g[0])

    # kmeans
    centroids = yt1.drop_duplicates()
    if init_method == 'centroids':
        init_method = Xt.loc[centroids.index]
    kmc = KMeans(n_clusters=len(centroids), init=init_method, n_init=10)
    kmc.fit(Xt)

    # predictions
    pt = pd.Series(kmc.predict(Xt), index=Xt.index)
    for ci in pt.unique():
        same_gen = pt[pt==ci].index.values
        #print('{} entries in cluster {}'.format(len(same_gen), ci))
        all_gen = np.concatenate(yt[same_gen].values)
        mode_gen, cont_gen = scipy.stats.mode(all_gen)
        pt.loc[same_gen] = mode_gen[0]
    terr = score(pt,yt)

    pv = pd.Series(kmc.predict(Xv), index=Xv.index)
    for ci in pv.unique():
        same_gen = pv[pv==ci].index.values
        mode_gen, cont_gen = scipy.stats.mode(np.concatenate(yv[same_gen].values))
        pv.loc[same_gen] = mode_gen[0]
    verr = score(pv,yv)

    if verbose:
        print_scores(terr, verr, 'KMeans')

    return verr

# ## Testing
# ### Initial Testing

# In[256]:

# Import data
n = None # None = 106,574
X = import_features(n)
y = import_tracks(n)
g = import_genres(n)

# format genres properly
cols = ['genres','genres_all']
y = y.loc[:,('track',cols)]
y.columns = cols
y = y.applymap(lambda t: json.loads(t))

# Remove entries with empty genres
eg = y.index[y['genres'].map(lambda t: len(t)==0)]
X = X.drop(eg)
y = y.drop(eg)

# Add another column holding parent genres
y['top_level'] = y['genres'].apply(lambda t: [g.loc[t[0]]['top_level']])
print('> Data imported successfully')

# In[ ]:

lda = m1_LDA(X, y['genres'], verbose=True)
lda = m1_LDA(X, y['genres_all'], verbose=True)
lda = m1_LDA(X, y['top_level'], verbose=True)

# In[ ]:

warnings.filterwarnings('default')
qda = m2_QDA(X, y['genres'], verbose=True)
qda = m2_QDA(X, y['genres_all'], verbose=True)
qda = m2_QDA(X, y['top_level'], verbose=True)

# In[ ]:

for km in ['k-means++', 'random', 'centroids']:
    print('> Testing method: ', km)
    m3_KMC(X, y['genres'], km, verbose=True)
    m3_KMC(X, y['genres_all'], km, verbose=True)
    m3_KMC(X, y['top_level'], km, verbose=True)

# ### Per Feature/Statistic Testing

# In[ ]:

yc = 'top_level'
cols = ['lda','qda','kmc-km++','kmc-rand','kmc-cent']
warnings.filterwarnings('ignore')

## test each feature
res_feat = pd.DataFrame(columns=cols)
for ft in X.columns.unique(level=0):
    print('> Testing feature: ', ft)
    res_feat.loc[ft, 'lda'] = m1_LDA(X.loc[:,pd.IndexSlice[ft, :, :]], y[yc])
    res_feat.loc[ft, 'qda'] = m2_QDA(X.loc[:,pd.IndexSlice[ft, :, :]], y[yc])
    res_feat.loc[ft, 'kmc-km++'] = m3_KMC(X.loc[:,pd.IndexSlice[ft, :, :]], y[yc], 'k-means++')
    res_feat.loc[ft, 'kmc-rand'] = m3_KMC(X.loc[:,pd.IndexSlice[ft, :, :]], y[yc], 'random')
    res_feat.loc[ft, 'kmc-cent'] = m3_KMC(X.loc[:,pd.IndexSlice[ft, :, :]], y[yc], 'centroids')

## test each statistic
res_stat = pd.DataFrame(columns=cols)
for st in X.columns.unique(level=1):
    print('> Testing statistic: ', st)
    res_stat.loc[st, 'lda'] = m1_LDA(X.loc[:,pd.IndexSlice[:, st, :]], y[yc])
    res_stat.loc[st, 'qda'] = m2_QDA(X.loc[:,pd.IndexSlice[:, st, :]], y[yc])
    res_stat.loc[st, 'kmc-km++'] = m3_KMC(X.loc[:,pd.IndexSlice[:, st, :]], y[yc], 'k-means++')
    res_stat.loc[st, 'kmc-rand'] = m3_KMC(X.loc[:,pd.IndexSlice[:, st, :]], y[yc], 'random')
    res_stat.loc[st, 'kmc-cent'] = m3_KMC(X.loc[:,pd.IndexSlice[:, st, :]], y[yc], 'centroids')

## print results
print('Results by feature:\n', res_feat)
print('Results by statistic:\n', res_stat)

## export results
res_feat.to_csv('res_feat_{}.csv'.format(yc))
res_stat.to_csv('res_stat_{}.csv'.format(yc))

# ### Final Testing

# In[296]:

m1_LDA(X.loc[:,pd.IndexSlice[:, :, :]], y['top_level'], verbose=True)
m1_LDA(X.loc[:,pd.IndexSlice[['chroma_cens','mfcc','spectral_contrast'], :, :]], y['top_level'], verbose=True)

qft = ['mfcc', 'spectral_bandwidth', 'spectral_centroid', 'spectral_contrast', 'spectral_rolloff', 'zcr']
m2_QDA(X.loc[:,pd.IndexSlice[:, ['std'], :]], y['top_level'], verbose=True)
m2_QDA(X.loc[:,pd.IndexSlice[qft, ['std'], :]], y['top_level'], verbose=True)
m2_QDA(X.loc[:,pd.IndexSlice[qft, ['mean','median','std'], :]], y['top_level'], verbose=True)

kft = ['mfcc', 'spectral_bandwidth', 'spectral_centroid', 'spectral_contrast', 'spectral_rolloff']
m3_KMC(X.loc[:,pd.IndexSlice[['mfcc'], ['skew'], :]], y['genres_all'], verbose=True)
m3_KMC(X.loc[:,pd.IndexSlice[['mfcc'], ['mean', 'median', 'skew'], :]], y['genres_all'], verbose=True)
m3_KMC(X.loc[:,pd.IndexSlice[['mfcc'], :, :]], y['genres_all'], verbose=True)

m3_KMC(X.loc[:,pd.IndexSlice[kft, ['skew'], :]], y['genres_all'], 'random', verbose=True)
m3_KMC(X.loc[:,pd.IndexSlice[kft, ['mean', 'median', 'skew'], :]], y['genres_all'], 'random', verbose=True)
m3_KMC(X.loc[:,pd.IndexSlice[kft, :, :]], y['genres_all'], 'random', verbose=True)

print('> Testing complete')


from .MetadataEnhancer import MetadataEnhancer
from utils import _try_for_key

class VariableEnhancer(MetadataEnhancer):

    def __init__(self, metadata: dict, enrichment_table: dict):
        super().__init__(metadata, enrichment_table)

    def enhance_metadata(self):
        """ enhance_metadata implementation for the variable enhancements.

        First the variables in the variableInformation metadata block are
        retrieved. Then for all variables we find enhancements using a table.
        We add all the retrieved enhancements to the cache with the var as key.
        Finally, we add the enhancements to the metadata.
        """
        variables = self.get_value_from_metadata('odisseiVariable',
                                                 'variableInformation')
        for variable_dict in variables:
            variable = _try_for_key(variable_dict,
                                    'odisseiVariableName.value')
            variable_uri = self.query_enrichment_table(variable)
            if variable_uri:
                self.add_enhancements_to_metadata(variable_uri, variable_dict)

    def add_enhancements_to_metadata(self, variable_uri: str,
                                     variable_dict: dict):
        """ Adds the variable enhancements to the metadata.

        If there are no enhancements, this method returns.
        Else it adds the first matched URI to the variable that was used to
        find the match.

        :param variable_uri: Enhancements to add to the metadata.
        :param variable_dict: The variable field to add the enhancements to.
        """
        variable_type_name = 'odisseiVariableVocabularyURI'
        self.add_enhancement_to_compound_metadata_field(variable_dict,
                                                        variable_type_name,
                                                        variable_uri)


#!/usr/bin/python

from __future__ import print_function
import atexit
from bcc import BPF
import os
from datetime import datetime

# load BPF program
b= BPF(src_file="kwtracer.c")

b.attach_kprobe(event="iov_iter_copy_from_user_atomic",fn_name="trace_do_user_space_write")
b.attach_kprobe(event="submit_bio", fn_name="trace_submit_bio")

date = datetime.today().strftime("%Y-%m-%d.%H:%M:%S")
old_trace = "./old_trace/tracelog_" + date + ".log"
print(old_trace)

os.system("sudo cp ./trace.log " + old_trace)

trace_file = open('./trace.log', 'w')

def print_event(cpu,data,size):
        event = b["events"].event(data)
        trace_line = "%s,%s\n" % (event.pid, event.comm)
        trace_file.write(trace_line)

b["events"].open_perf_buffer(print_event)
while 1:
	try :
		b.perf_buffer_poll()
	except KeyboardInterrupt:
                trace_file.close()
                exit()


# -*- coding: utf-8 -*-
"""
Created on Sun Nov 25 12:08:01 2018

@author: Jim
"""
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import  linear_model
"""
read file
"""
csvfile = "Concrete_Data.csv"
data = pd.read_csv(csvfile)
(row,column)=data.shape
X_train=data['Age (day)'] #input
X_trainsecond=np.zeros([row,2])
for i in range(row):
    X_trainsecond[i][0]=X_train[i]
    X_trainsecond[i][1]=0
Y_train=data['Concrete compressive strength(MPa, megapascals) '] #output
#print(X_trainsecond.shape,Y_train.shape)

"""
lineaer regression
"""
regr = linear_model.LinearRegression()
regr.fit(X_trainsecond,Y_train)
print('weight: ',regr.coef_)
print('bias: ',regr.intercept_)
print('accuracy(r2_score): ',regr.score(X_trainsecond,Y_train))
"""
visualization, scatter plot and linear regression expresiion
"""
x=([0,0],[0,0])
x[0][0]=min(X_train)
x[1][0]=max(X_train)
y=([0,0])
y[0] = regr.coef_*x[0] + regr.intercept_
y[1] = regr.coef_*x[1] + regr.intercept_
plt.scatter(X_train,Y_train)
plt.plot(x,y)
plt.show()


import requests
import json
from dotenv import load_dotenv
import os
import ipdb
import traceback
from datetime import datetime
import pandas as pd
import matplotlib.pyplot as plt

load_dotenv()

notion_token = os.environ.get("NOTION_TOKEN")
database_id = os.environ.get("NOTION_DB_ID")
headers = {
    "Authorization": "Bearer " + notion_token,
    "Content-Type": "application/json",
    "Notion-Version": "2021-05-13"
}

def get_lap_time(result, key):
    if len(result['properties'][f'{key}']['rich_text']) > 0:
        value = result['properties'][f'{key}']['rich_text'][0]['plain_text']
        return value
    else:
        return '00:00'

def convert_db_to_dict(db_json):
    results = db_json['results']

    db_dict = []
    for result in results:
        try:
            temp = {
                'date': result['properties']['Date']['title'][0]['plain_text'],
                'lap1': get_lap_time(result, 'Lap 1'),
                'lap2': get_lap_time(result, 'Lap 2'),
                'lap3': get_lap_time(result, 'Lap 3'),
                'sprint': get_lap_time(result, 'Sprint'),
                'set1': result['properties'].get('Set 1', {}).get('number', 0),
                'set2': result['properties'].get('Set 2', {}).get('number', 0),
                'set3': result['properties'].get('Set 3', {}).get('number', 0)
            }
            # db_dict[result['properties']['Date']['title'][0]['plain_text']] = temp
            db_dict.append(temp)
        except:
            print(traceback.print_exc())

    db_dict.sort(key = lambda x: datetime.strptime(x['date'], '%d-%b-%Y'))
    return db_dict

def dict_to_df(db_dict):
    df = pd.DataFrame(db_dict)
    df['total running'] = pd.to_timedelta('00:' + df['lap1']) + pd.to_timedelta('00:' + df['lap2']) + pd.to_timedelta('00:' + df['lap3']) + pd.to_timedelta('00:' + df['sprint'])
    df['total running'] = df['total running'].dt.seconds.div(60).astype(int).astype(str).str.zfill(2) + ':' + df['total running'].dt.seconds.mod(60).astype(str).str.zfill(2)
    df['total pushups'] = df['set1'] + df['set2'] + df['set3']
    return df

# https://prettystatic.com/notion-api-python/
def read_db(database_id, headers):
    url = f"https://api.notion.com/v1/databases/{database_id}/query"

    res = requests.request("POST", url, headers=headers)
    data = res.json()
    # print(res.status_code)

    # with open('./db.json', 'w', encoding='utf8') as f:
    #     json.dump(data, f, ensure_ascii=False)
    return data

def plot_data(df):
    # Convert lap times to seconds
    df['lap1_sec'] = pd.to_timedelta('00:' + df['lap1']).dt.total_seconds()
    df['lap2_sec'] = pd.to_timedelta('00:' + df['lap2']).dt.total_seconds()
    df['lap3_sec'] = pd.to_timedelta('00:' + df['lap3']).dt.total_seconds()
    df['sprint_sec'] = pd.to_timedelta('00:' + df['sprint']).dt.total_seconds()

    # Calculate total running time and total pushups
    df['total_running_sec'] = df['lap1_sec'] + df['lap2_sec'] + df['lap3_sec'] + df['sprint_sec']
    df['total_pushups'] = df['set1'] + df['set2'] + df['set3']

    # Plot the data
    fig, ax = plt.subplots()
    df.plot(x='date', y='total_running_sec', kind='bar', ax=ax)
    df.plot(x='date', y='total_pushups', kind='line', ax=ax, secondary_y=True)
    ax.set_xlabel('Date')
    ax.set_ylabel('Total Running Time (seconds)')
    ax.right_ax.set_ylabel('Total Pushups')
    plt.show()

data = read_db(database_id, headers)
db_dict = convert_db_to_dict(data)
db_df = dict_to_df(db_dict)
plot_data(db_df)
# print(db_df)


# This file is executed on every boot (including wake-boot from deepsleep)
import esp
esp.osdebug(None)
#import webrepl
# webrepl.start()

from lib.wifiManager.wifiManager import WifiManager

wifiM = WifiManager()

if wifiM.connect():
    print("********WIFI is Connected**********")
else:
    wifiM.createAP()
    print("********WIFI as AccessPoint********")


import boto3
from botocore.exceptions import ClientError
import pytest

from framework_from_conformance_pack import ConformancePack

@pytest.mark.parametrize(
    "in_name, error_code",
    [("test-name", None), ("garbage", None), ("test-name", "TestException")],
)
def test_get_conformance_pack(make_stubber, monkeypatch, in_name, error_code):
    config_client = boto3.client("config")
    config_stubber = make_stubber(config_client)
    pack = ConformancePack(config_client, None)
    cpack_name = "test-name"

    monkeypatch.setattr("builtins.input", lambda x: in_name)

    config_stubber.stub_describe_conformance_packs([cpack_name], error_code=error_code)

    if error_code is None:
        if in_name != "garbage":
            got_cpack_name = pack.get_conformance_pack()
            assert got_cpack_name == cpack_name
        else:
            with pytest.raises(Exception):
                pack.get_conformance_pack()
    else:
        with pytest.raises(ClientError) as exc_info:
            pack.get_conformance_pack()
        assert exc_info.value.response["Error"]["Code"] == error_code

def test_create_custom_controls(make_stubber):
    config_client = boto3.client("config")
    config_stubber = make_stubber(config_client)
    auditmanager_client = boto3.client("auditmanager")
    auditmanager_stubber = make_stubber(auditmanager_client)
    pack = ConformancePack(config_client, auditmanager_client)
    pack_name = "test-pack_name"
    rule_names = ["rule-1", "rule-2"]
    source_ids = ["src-1", "src-2"]
    control_ids = [f'ctl-{"1"*36}', f'ctl-{"2"*36}']

    config_stubber.stub_describe_conformance_pack_compliance(pack_name, rule_names)
    for rule_name, source_id, control_id in zip(rule_names, source_ids, control_ids):
        config_stubber.stub_describe_config_rules([rule_name], source_ids=[source_id])
        auditmanager_stubber.stub_create_control(
            f"Config-{rule_name}", source_id, control_id
        )

    got_control_ids = pack.create_custom_controls(pack_name)
    assert [got["id"] for got in got_control_ids] == control_ids

@pytest.mark.parametrize("error_code", [None, "TestException"])
def test_create_custom_framework(make_stubber, error_code):
    auditmanager_client = boto3.client("auditmanager")
    auditmanager_stubber = make_stubber(auditmanager_client)
    pack = ConformancePack(None, auditmanager_client)
    pack_name = "test-pack_name"
    control_ids = [{"id": f'ctl-{"1"*36}'}, {"id": f'ctl-{"2"*36}'}]
    framework = {"name": f"Config-Conformance-pack-{pack_name}", "id": f'fw-{"1"*36}'}

    auditmanager_stubber.stub_create_assessment_framework(
        framework["name"],
        [{"name": pack_name, "controls": control_ids}],
        framework["id"],
        error_code=error_code,
    )

    if error_code is None:
        pack.create_custom_framework(pack_name, control_ids)
    else:
        with pytest.raises(ClientError) as exc_info:
            pack.create_custom_framework(pack_name, control_ids)
        assert exc_info.value.response["Error"]["Code"] == error_code


import logging
import sys

from telemetry.value import histogram
from telemetry.value import histogram_util
from telemetry.value import scalar

from metrics import Metric

_HISTOGRAMS = [
    {
        'name': 'V8.MemoryExternalFragmentationTotal', 'units': 'percent',
        'display_name': 'V8_MemoryExternalFragmentationTotal',
        'type': histogram_util.RENDERER_HISTOGRAM,
        'description': 'Total external memory fragmentation after each GC in '
                       'percent.',
    },
    {
        'name': 'V8.MemoryHeapSampleTotalCommitted', 'units': 'kb',
        'display_name': 'V8_MemoryHeapSampleTotalCommitted',
        'type': histogram_util.RENDERER_HISTOGRAM,
        'description': 'The total size of committed memory used by V8 after '
                       'each GC in KB.'
    },
    {
        'name': 'V8.MemoryHeapSampleTotalUsed', 'units': 'kb',
        'display_name': 'V8_MemoryHeapSampleTotalUsed',
        'type': histogram_util.RENDERER_HISTOGRAM,
        'description': 'The total size of live memory used by V8 after each '
                       'GC in KB.',
    },
    {
        'name': 'V8.MemoryHeapSampleMaximumCommitted', 'units': 'kb',
        'display_name': 'V8_MemoryHeapSampleMaximumCommitted',
        'type': histogram_util.RENDERER_HISTOGRAM
    },
    {
        'name': 'Memory.RendererUsed', 'units': 'kb',
        'display_name': 'Memory_RendererUsed',
        'type': histogram_util.RENDERER_HISTOGRAM
    },
    {
        'name': 'Memory.BrowserUsed', 'units': 'kb',
        'display_name': 'Memory_BrowserUsed',
        'type': histogram_util.BROWSER_HISTOGRAM
    },
]

class MemoryMetric(Metric):
  """MemoryMetric gathers memory statistics from the browser object.

  This includes both per-page histogram stats, most about javascript
  memory usage, and overall memory stats from the system for the whole
  test run."""

  def __init__(self, browser):
    super(MemoryMetric, self).__init__()
    self._browser = browser
    start_memory_stats = self._browser.memory_stats
    self._start_commit_charge = None
    if 'SystemCommitCharge' in start_memory_stats:
      self._start_commit_charge = start_memory_stats['SystemCommitCharge']
    self._memory_stats = None
    self._histogram_start = dict()
    self._histogram_delta = dict()
    self._started = False

  @classmethod
  def CustomizeBrowserOptions(cls, options):
    options.AppendExtraBrowserArgs([
        '--enable-stats-collection-bindings',
        # For a hard-coded set of Google pages (such as GMail), we produce
        # custom memory histograms (V8.Something_gmail) instead of the generic
        # histograms (V8.Something), if we detect that a renderer is only
        # rendering this page and no other pages. For this test, we need to
        # disable histogram customizing, so that we get the same generic
        # histograms produced for all pages.
        '--disable-histogram-customizer'
    ])

  def Start(self, page, tab):
    """Start the per-page preparation for this metric.

    Here, this consists of recording the start value of all the histograms.
    """
    if not self._browser.supports_memory_metrics:
      logging.warning('Memory metrics not supported.')
      return

    self._started = True

    for h in _HISTOGRAMS:
      histogram_data = histogram_util.GetHistogram(
          h['type'], h['name'], tab)
      # Histogram data may not be available
      if not histogram_data:
        continue
      self._histogram_start[h['name']] = histogram_data

  def Stop(self, page, tab):
    """Prepare the results for this page.

    The results are the differences between the current histogram values
    and the values when Start() was called.
    """
    if not self._browser.supports_memory_metrics:
      return

    assert self._started, 'Must call Start() first'
    for h in _HISTOGRAMS:
      # Histogram data may not be available
      if h['name'] not in self._histogram_start:
        continue
      histogram_data = histogram_util.GetHistogram(
          h['type'], h['name'], tab)
      self._histogram_delta[h['name']] = histogram_util.SubtractHistogram(
          histogram_data, self._histogram_start[h['name']])

  # Optional argument trace_name is not in base class Metric.
  # pylint: disable=arguments-differ
  def AddResults(self, tab, results, trace_name=None):
    """Add results for this page to the results object."""
    if not self._browser.supports_memory_metrics:
      return

    assert self._histogram_delta, 'Must call Stop() first'
    for h in _HISTOGRAMS:
      # Histogram data may not be available
      if h['name'] not in self._histogram_start:
        continue
      results.AddValue(histogram.HistogramValue(
          results.current_page, h['display_name'], h['units'],
          raw_value_json=self._histogram_delta[h['name']], important=False,
          description=h.get('description')))
    self._memory_stats = self._browser.memory_stats
    if not self._memory_stats['Browser']:
      return
    AddResultsForProcesses(results, self._memory_stats,
                           metric_trace_name=trace_name)

    if self._start_commit_charge:
      end_commit_charge = self._memory_stats['SystemCommitCharge']
      commit_charge_difference = end_commit_charge - self._start_commit_charge
      results.AddValue(scalar.ScalarValue(
          results.current_page,
          'commit_charge.' + (trace_name or 'commit_charge'),
          'kb', commit_charge_difference, important=False,
          description='System commit charge (committed memory pages).'))
    results.AddValue(scalar.ScalarValue(
        results.current_page, 'processes.' + (trace_name or 'processes'),
        'count', self._memory_stats['ProcessCount'], important=False,
        description='Number of processes used by Chrome.'))

def AddResultsForProcesses(results, memory_stats, chart_trace_name='final',
                           metric_trace_name=None,
                           exclude_metrics=None):
  """Adds memory stats for browser, renderer and gpu processes.

  Args:
    results: A telemetry.results.PageTestResults object.
    memory_stats: System memory stats collected.
    chart_trace_name: Trace to identify memory metrics. Default is 'final'.
    metric_trace_name: Trace to identify the metric results per test page.
    exclude_metrics: List of memory metrics to exclude from results,
                     e.g. VM, WorkingSetSize, etc.
  """
  metric = 'resident_set_size'
  if sys.platform == 'win32':
    metric = 'working_set'

  exclude_metrics = exclude_metrics or {}

  def AddResultsForProcessTypes(process_types_memory, process_type_trace):
    """Add all results for a given set of process types.

    Args:
      process_types_memory: A list of process types, e.g. Browser, 'Renderer'.
      process_type_trace: The name of this set of process types in the output.
    """
    def AddResult(value_name_memory, value_name_trace, description):
      """Add a result for a given statistic.

      Args:
        value_name_memory: Name of some statistic, e.g. VM, WorkingSetSize.
        value_name_trace: Name of this statistic to be used in the output.
      """
      if value_name_memory in exclude_metrics:
        return
      if len(process_types_memory) > 1 and value_name_memory.endswith('Peak'):
        return
      values = []
      for process_type_memory in process_types_memory:
        stats = memory_stats[process_type_memory]
        if value_name_memory in stats:
          values.append(stats[value_name_memory])
      if values:
        if metric_trace_name:
          current_trace = '%s_%s' % (metric_trace_name, process_type_trace)
          chart_name = value_name_trace
        else:
          current_trace = '%s_%s' % (value_name_trace, process_type_trace)
          chart_name = current_trace
        results.AddValue(scalar.ScalarValue(
            results.current_page, '%s.%s' % (chart_name, current_trace), 'kb',
            sum(values) / 1024, important=False, description=description))

    AddResult('VM', 'vm_%s_size' % chart_trace_name,
              'Virtual Memory Size (address space allocated).')
    AddResult('WorkingSetSize', 'vm_%s_%s_size' % (metric, chart_trace_name),
              'Working Set Size (Windows) or Resident Set Size (other '
              'platforms).')
    AddResult('PrivateDirty', 'vm_private_dirty_%s' % chart_trace_name,
              'Private Dirty is basically the amount of RAM inside the '
              'process that can not be paged to disk (it is not backed by the '
              'same data on disk), and is not shared with any other '
              'processes. Another way to look at this is the RAM that will '
              'become available to the system when that process goes away '
              '(and probably quickly subsumed into caches and other uses of '
              'it).')
    AddResult('ProportionalSetSize',
              'vm_proportional_set_size_%s' % chart_trace_name,
              'The Proportional Set Size (PSS) number is a metric the kernel '
              'computes that takes into account memory sharing -- basically '
              'each page of RAM in a process is scaled by a ratio of the '
              'number of other processes also using that page. This way you '
              'can (in theory) add up the PSS across all processes to see '
              'the total RAM they are using, and compare PSS between '
              'processes to get a rough idea of their relative weight.')
    AddResult('SharedDirty', 'vm_shared_dirty_%s' % chart_trace_name,
              'Shared Dirty is the amount of RAM outside the process that can '
              'not be paged to disk, and is shared with other processes.')
    AddResult('VMPeak', 'vm_peak_size',
              'The peak Virtual Memory Size (address space allocated) usage '
              'achieved by the * process.')
    AddResult('WorkingSetSizePeak', '%s_peak_size' % metric,
              'Peak Working Set Size.')

  AddResultsForProcessTypes(['Browser'], 'browser')
  AddResultsForProcessTypes(['Renderer'], 'renderer')
  AddResultsForProcessTypes(['Gpu'], 'gpu')
  AddResultsForProcessTypes(['Browser', 'Renderer', 'Gpu'], 'total')


import neo4j.exceptions

from reporting import user_watcher

def test_watch_users(mocker):
    runner = user_watcher.app.test_cli_runner()
    mocker.patch(
        "reporting.user_watcher._is_shutdown",
        side_effect=[False, False, True, True],
    )
    bootstrap_mock = mocker.patch("reporting.user_watcher._bootstrap")
    delete_mock = mocker.patch(
        "reporting.services.reporting_neo4j.delete_expired_users",
        side_effect=[neo4j.exceptions.ServiceUnavailable, None],
    )
    sleep_mock = mocker.patch("time.sleep")
    runner.invoke(user_watcher.watch_users)
    assert bootstrap_mock.call_count == 1
    assert sleep_mock.call_count == 1
    assert delete_mock.call_count == 1


from django.urls import path
from . import views

app_name = 'blog'

urlpatterns = [
    path('', views.MainListView.as_view(), name='index'),
    path('about/', views.AboutUsView.as_view(), name='about'),
    path('feedback/', views.FeedBackFormView.as_view(), name='feedback'),
    path('feedback/success/', views.FeedBackSuccessView.as_view(), name='feedback_success'),
    path('terms_conditions/', views.TermsConditionsView.as_view(), name='terms_conditions'),
    path('add_post/', views.AddPostView.as_view(), name='add_post'),
    path('search/', views.SearchListView.as_view(), name='search'),
    path('<slug:slug>', views.PostDetailView.as_view(), name='post_detail'),
    path('<slug:slug>/edit/', views.EditPostView.as_view(), name='edit_post'),
    path('tag/<slug:slug>', views.TagListView.as_view(), name='tag')
]


# -*- coding: utf-8 -*-
import sys
sys.dont_write_bytecode = True
import os
import torch
from core.config import Config
from core import Test

PATH = "/.../.../...-bgl_time-tcniniNet-2-5-Feb-27-2023-18-07-28"
VAR_DICT = {
    "test_epoch": 5,
    "device_ids": "0",
    "inner_train_iter": 100,
    "n_gpu": 1,
    "test_episode": 100,
    "batch_size": 64,
    "episode_size": 1,
    # "query_num": 20,
    "test_shot": 5,
    # "shot_num": 20,
}

def main(rank, config):
    test = Test(rank, config, PATH)
    test.test_loop()

if __name__ == "__main__":
    config = Config(os.path.join(PATH, "config.yaml"), VAR_DICT).get_config_dict()
    if config["n_gpu"] > 1:
        os.environ["CUDA_VISIBLE_DEVICES"] = config["device_ids"]
        torch.multiprocessing.spawn(main, nprocs=config["n_gpu"], args=(config,))
    else:
        main(0, config)


#!/usr/bin/env python
# coding: utf-8

# # Raster data analysis
#
# Raster data represent a matrix of cells (or pixels) organized into rows and columns (or a grid). Grid cells can represent data that changes **continuously** across a landscape (surface) such as elevation, air temperature, or . reflectance data from satellite imaging platforms. Grid cells can also represent **discrete** data such as vegetation type or land cover.
#
# ```{image} images/rainier_adams.jpg
# :alt: rainier_adams
# :width: 600px
# :align: center
# ```
#
#
# We recommend three libraries for accessing and analyzing raster data in Python. The first is called `rasterio` which builds on the popular **Geographic Raster Abstraction Library** or `GDAL`. It supports read/write access for over 160 raster formats (e.g. GeoTIFF, NetCDF4) and includes methods for finding dataset information, reprojections, resampling, format conversion, and mosaicking. Once we have imported, resampled the data etc., we can apply fast matrix operations using `NumPy`. Finally, we may also use `xarray` which introduces labels in the form of **dimensions**, **coordinates** and **attributes** on top of raw NumPy-like arrays, a bit like `Pandas`.
#
# ```{image} images/raster_matrix.png
# :alt: raster matrix
# :width: 500px
# :align: center
# ```

# ## Review of raster data
#
# * Raster data represent a matrix of cells (or pixels) organized into rows and columns (or a grid)
#
# ```{image} images/raster_concept.png
# :alt: raster_concept
# :width: 500px
# :align: center
# ```

# ## Examples: surface maps
#
# * Grid cells can represent data that changes **continuously** across a landscape (surface) such as elevation or air temperature.
#
# ```{image} images/elevation.gif
# :alt: elevation
# :width: 400px
# :align: center
# ```

# ## Examples: satellite imagery
#
# * Grid cells can represent from a satellite imagaing platforms such as reflectance.
#
# ```{image} images/satellite.gif
# :alt: satellite
# :width: 400px
# :align: center
# ```

# ### Examples: classification maps
#
# * Grid cells can also represent **discrete** data (e.g. vegetation type or land cover).
#
# ```{image} images/classification.gif
# :alt: elevation
# :width: 600px
# :align: center
# ```

# In this demo, we will be working with elevation data, also known as a Digital Elevation Model (DEM), of the Cascades Mountain Range that includes Mt. Rainier and Mt. Adams.
#
# The data is formatted as a `GeoTIFF` and we will open it using `rasterio` function, `open()`. This function take a **path string** and returns a **dataset object**.

# In[11]:

import rasterio
import numpy as np

src = rasterio.open('data/N46W122.tif')
src

# ````{margin}
# ```{note}
# `src` stands for **source**
# ```
# ````

# ## Dataset attributes
#
# The **dataset object** contains a number of **attributes** which can be explored using the following methods. Remember that a raster **band** is an array of values representing **a single** variable in 2D space. All bands of a dataset have the **same** number of rows and columns.

# In[12]:

print(f"Number of bands: {src.count}")
print(f"Width: {src.width}")
print(f"Height: {src.height}")

# ## Georeferencing
#
# Like vector data, pixels in raster data can be mapped to regions on the Earth's surface. Like `GeoPandas`, we can display the **coordinate reference system** of our data using the `crs` method.

# In[13]:

src.crs

# Now that we know our data has a WGS84 geographic projection (i.e. longitudes and latitudes), we can display the **extent** of our dataset using the `bounds` method.

# In[14]:

src.bounds

# Finally, we can display the  dataset's geospatial transform using the `transform` method. This function displays similar information to `bounds` but also contains the **spatial resolution** of the dataset (i.e. the dimensions that each pixel of our dataset represents on the ground). Since our dataset has a **WGS84 geographic projection** (i.e. `EPSG:4326`), the units of spatial resolution are in **degrees**.

# In[15]:

src.transform

# ## Reading raster data
#
# Now that we have some basic information about our data, we can go ahead and import it using the `read()` function. Data from a raster band can be accessed by the band's index number. Note that bands are indexed from 1 due to a  GDAL convention.

# In[16]:

srtm = src.read(1)

# ````{margin}
# ```{note}
# `srtm` stands for the [**Shuttle Radar Topography Mission**](https://www2.jpl.nasa.gov/srtm/) which collected this elevation data.
# ```
# ````

# The read() method returns a numpy N-D array.

# In[17]:

srtm

# In[18]:

type(srtm)

# We can have a look at the data using `matplotlib`.

# In[19]:

import matplotlib.pyplot as plt

# Plot data
fig, ax = plt.subplots(figsize=(8,8))
im = ax.imshow(srtm)
ax.set_title("Mt Rainier and Mt Adams", fontsize=14)
cbar = fig.colorbar(im, orientation='vertical')
cbar.ax.set_ylabel('Elevation', rotation=270, fontsize=14)
cbar.ax.get_yaxis().labelpad = 20

# ## Indexing
#
# Many GIS tasks require us to read raster values at given locations. Rasterio dataset objects have an `index()` method for deriving the **array indices** corresponding to points in **georeferenced space**.
#
# Let's demonstrate with an example... what is the elevation of the summit of Mt Rainier? (`-121.760424, 46.852947`)

# In[20]:

# Define latitude and longitude of summit
rainier_summit = [-121.760424, 46.852947]

# Find row/column in corresponding raster dataset
loc_idx = src.index(rainier_summit[0], rainier_summit[1])

print(f"Grid cell index: {loc_idx}")

# We can use **matrix indexing** to find the value of the raster data at that location (see [Week 2 demo](../02a-demo.ipynb#Matrix-indexing-and-slicing) for reminder).

# In[21]:

elevation = srtm[loc_idx]

print(f"The summit of Mt Rainier is at {int(elevation)} m or {int(elevation * 3.281)} feet")

# In[22]:

fig, ax = plt.subplots(figsize=(8,8))
im = ax.imshow(srtm)

# Plot a point on grid
ax.scatter(loc_idx[1], loc_idx[0], s=50, color='red')

ax.set_title("Mt Rainier and Mt Adams", fontsize=14)
cbar = fig.colorbar(im, orientation='vertical')
cbar.ax.set_ylabel('Elevation', rotation=270, fontsize=14)
cbar.ax.get_yaxis().labelpad = 20

# ## More indexing methods
#
# How would we find the index of the **lowest elevation** in this raster dataset? The `NumPy` [`argmin()`](https://numpy.org/doc/stable/reference/generated/numpy.argmin.html) function returns the indices of the minimum values of an array.

# In[23]:

min_idx_value = srtm.argmin()
print(min_idx_value)

# Wait... I thought this dataset has two dimensions... Yes but by default, `argmin()` returns the index as a flattened (1D) array. Fortunately, converting from 1D back to 2D is simple using `np.unravel_index`.

# In[24]:

low_idx = np.unravel_index(min_idx_value, srtm.shape)
print(low_idx)

# In[25]:

elevation = srtm[low_idx]

print(f"The lowest elevation is {elevation} m")

# In[26]:

fig, ax = plt.subplots(figsize=(7,7))
im = ax.imshow(srtm)

# Plot a point on grid
ax.scatter(loc_idx[1], loc_idx[0], s=50, color='red')
ax.scatter(low_idx[1], low_idx[0], s=50, color='orange')

ax.set_title("Mt Rainier and Mt Adams", fontsize=14)
cbar = fig.colorbar(im, orientation='vertical')
cbar.ax.set_ylabel('Elevation', rotation=270, fontsize=14)
cbar.ax.get_yaxis().labelpad = 20

# ## Converting between indices and coordinates
#
# Since `Rasterio` knows the spatial reference system of our data, we can convert between indices (i.e. rows/columns) and coordinates (i.e. lats/longs) very conveniently. To find the latitude and longitude of the lowest elevation grid cell we can use the [`transform`](https://rasterio.readthedocs.io/en/latest/api/rasterio.transform.html) module.
#
# `transform.xy`, for example, takes the **dataset transform** along with the **row and column index** and converts them to x and y values in the dataset's coordinate reference system.

# In[31]:

rasterio.transform.xy(src.transform, low_idx[0], low_idx[1])

# Inversely, `transform.rowcol` takes the **dataset transform** along with the **x and y values in the coordinate reference system** and converts them to a row and column indices.

# In[32]:

rasterio.transform.rowcol(src.transform, rainier_summit[0], rainier_summit[1])

# ```{note}
# Note that `transform.rowcol` provides the same output as `src.index` used earlier.
# ```

# ## Reprojecting
#
# We could use `Rasterio` to reproject raster data... but it’s quite tricky!
#
# ```{image} images/rasterio_reproject.png
# :alt: rasterio reproject
# :width: 700px
# :align: center
# ```
#
# Instead we recommend using [**GDAL utilities**](https://gdal.org/programs/index.html#raster-programs). We can execute these commands in our jupyter notebook cells using the `!` sign.
#
# To reproject our data, we can use [`gdalwarp`](https://gdal.org/programs/gdalwarp.html#gdalwarp). All we need to do is set a **target spatial reference** using the `-t_srs` flag followed by a space, the **input dataset**, and the **output dataset**. Below we set the target spatial reference to [**UTM Zone 10N**](https://epsg.io/32610) (or EPSG:32610) which is the UTM Zone for the Pacific Northwest.
#
# ```{image} images/utm_zones.png
# :alt: utm zones
# :width: 300px
# :align: center
# ```

# In[17]:

get_ipython().system('gdalwarp -t_srs EPSG:32610 data/N46W122.tif data/N46W122_utm.tif')

# If we navigate to our `data` folder we should see a new file called `N46W122_utm.tif`. Let's `open` this new GeoTIFF and check that is has a new projection.

# In[18]:

src = rasterio.open('data/N46W122_utm.tif')
src.crs

# In[19]:

srtm = src.read(1)

fig, ax = plt.subplots(figsize=(7,7))
im = ax.imshow(srtm)

ax.set_title("Mt Rainier and Mt Adams", fontsize=14)
cbar = fig.colorbar(im, orientation='vertical')
cbar.ax.set_ylabel('Elevation', rotation=270, fontsize=14)
cbar.ax.get_yaxis().labelpad = 20

# Why does the data look so strange now? Well, since we reprojected it, our data no longer represents a rectangle/square. Since all arrays have to be rectangles/squares, our reprojection introduced some **NoData values** at the edges. If we have a look at our array, we see that these NoData values are indicated by the integer `-32768` which is the smallest possible value that can be represented by the `int16` data type (i.e. -32,768 to 32,767).

# In[20]:

srtm

# We can mask these NoData values by using NumPy's [**masked array**](https://numpy.org/doc/stable/reference/maskedarray.generic.html) module that makes it easier to deal with arrays that have missing or invalid entries.

# In[21]:

srtm_masked = np.ma.masked_array(srtm, mask=(srtm == -32768))

# Now when we plot the data, the NoData values are not assigned a color.`

# In[22]:

fig, ax = plt.subplots(figsize=(7,7))
im = ax.imshow(srtm_masked)

ax.set_title("Mt Rainier and Mt Adams", fontsize=14)
cbar = fig.colorbar(im, orientation='vertical')
cbar.ax.set_ylabel('Elevation', rotation=270, fontsize=14)
cbar.ax.get_yaxis().labelpad = 20

# ## Resampling
#
# GDAL utilites make it straightforward to change the **spatial resolution** of our raster dataset. To reduce the pixel size of our dataset from around 30 m to 1,000 m can be carried out using `gdalwarp`. This time, however, we specify the `-tr` flag, which stands for **target resolution**, followed by the pixel size we want.

# In[25]:

get_ipython().system('gdalwarp -tr 1000 -1000 data/N46W122_utm.tif data/N46W122_utm_1000.tif')

# Now when we `open`, `read`, `mask`, and `plot` our data, we will see that it looks a lot coarser/pixelated because  each grid cell represents 1 km on the ground.

# In[26]:

# Open new raster dataset
src = rasterio.open('data/N46W122_utm_1000.tif')

# Read new raster dataset
srtm_1000 = src.read(1)

# Mask data
srtm_1000_masked = np.ma.masked_array(srtm_1000, mask=(srtm_1000 == -32768))

# Plot
fig, ax = plt.subplots(figsize=(8,8))
im = ax.imshow(srtm_1000_masked)

ax.set_title("Mt Rainier and Mt Adams", fontsize=14)
cbar = fig.colorbar(im, orientation='vertical')
cbar.ax.set_ylabel('Elevation', rotation=270, fontsize=14)
cbar.ax.get_yaxis().labelpad = 20


# import library and abbreviate pyplot for easier use
import matplotlib.pyplot as plt

# list of data we'll be plotting
inputValues = [1, 2, 3, 4, 5]
squares = [1, 4, 9, 16, 25]

# using a built-in style
    # note: this needs to go BEFORE running subplots() and plot()
plt.style.use('dark_background')

# fig - represents entire figure (collection of plots generated)
# ax - represents a single plot (variable we'll use most of the time)
# subplots() - generate one or more plots in the same figure
fig, ax = plt.subplots()

# plot() - will try to plot data its given in a meaningful way
# linewidth=3 argument controlling the line thickness of our plot
ax.plot(inputValues, squares, linewidth=3)

# Set chart title and label axes
ax.set_title("Square Numbers", fontsize=24)
ax.set_xlabel("Value", fontsize=14)
ax.set_ylabel("Square of Value", fontsize=14)

# Set size of tick labels
ax.tick_params(axis='both', labelsize=14)

# show() opens matplotlib's viewer and displays the plot
plt.show()


from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from urllib.parse import quote
import sys
from pyquery import PyQuery as pq
import re
from selenium.webdriver.support.select import Select
import time
import random
import config
import save
from lxml import etree

def get_match_info(year, league, stage, text):
    match = {
        'year': year,
        'stage': stage,
        'league': league,
    }

    home_name = text[2].lstrip('21').split('[')
    away_name = text[4].lstrip('21').split('[')
    match['date'] = text[1].replace("\n", " ", 1)
    match['home_name'] = home_name[0]
    match['away_name'] = away_name[0]
    match['score'] = text[3]
    match['half_score'] = text[10]

    return match

def get_odds(url):
    odds_browser = config.get_webdriver()
    odds_wait = WebDriverWait(odds_browser, 10)

    try:
        odds_browser.get(url)
        odds_wait.until(
                EC.presence_of_element_located((By.CSS_SELECTOR, '#sel_showType')))

        select = Select(odds_browser.find_element_by_id("sel_showType"))
        select.select_by_value('1')
        odds_wait.until(
                EC.presence_of_element_located((By.CSS_SELECTOR, '#dataList')))

        html = etree.HTML(odds_browser.page_source)
        doc = pq(html)
        td = doc('#oddstr_281 td')

        text = []
        odds = {}
        for item in td.items():
            text.append(item.text())
        odds['start'] = text[2:5]

        text = []
        end_text = []
        next = td.parent().next()
        for item in next.items():
            end_text.append(item.text())
        text = end_text[0].split()
        odds['end'] = text[0:3]

    except TimeoutException:
        get_odds(url)
    time.sleep(random.randint(3, 5))
    odds_browser.close()
    return odds

def get_over_down(url):
    odds_browser = config.get_webdriver()
    odds_wait = WebDriverWait(odds_browser, 10)

    try:
        odds_browser.get(url)
        odds_wait.until(
            EC.presence_of_element_located((By.CSS_SELECTOR, '#odds')))

        html = etree.HTML(odds_browser.page_source)
        doc = pq(html)
        tr = doc('#odds tbody tr')

        text = []
        odds = {}
        for item in tr.items():
            for td in item.find('td').items():
                result = re.search('Bet365', td.text())
                if result:
                    bet365 = td.parent()
                    for bet365_td in bet365.find('td').items():
                        text.append(bet365_td.text())
                    break

        if len(text) < 10:
            print("大小球赔率页面错误")
            sys.exit()
        odds['start'] = text[2:5]
        odds['end'] = text[8:11]

    except TimeoutException:
        get_over_down(url)
    time.sleep(random.randint(3, 5))
    odds_browser.close()
    return odds

def get_match(page, year, league, stage):
    html = etree.HTML(page.page_source)
    doc = pq(html)
    has_odds = False
    has_OD = False

    items = doc('#Table3 tr a').parent().parent()

    for item in items.items():
        text = []
        tds = item.find('td')
        a = tds.find('a')
        for td in tds.items():
            text.append(td.text())

        if len(text) == 11:
            if text[3]:
                match = get_match_info(year, league, stage, text)
                is_exist = save.get_exist_state(match)
                if is_exist == 'existed':
                    continue
            else:
                return 'no_start'

            for item in a.items():
                url = item.attr('href')

                result = re.search('oddslist', url)
                if result:
                    has_odds = True
                    odds = []
                    odds = get_odds(url)
                    match['odds'] = odds

                result = re.search('OverDown', url)
                if result:
                    has_OD = True
                    over_down = get_over_down(url)
                    match['over_down'] = over_down

            if match:
                print(match)
                save.save_match(match)
        elif len(text) != 0:
            print("比赛数据错误")
            sys.exit()
    return match


import torch
import numpy as np
import torch.nn as nn
from itertools import product, permutations

try:
    from clarity.enhancer.compressor import CompressorTorch
    from clarity.enhancer.nalr import NALRTorch
    LIB_CLARITY = True
except ModuleNotFoundError:
    print("There's no clarity library")
    LIB_CLARITY = False

def loss_sisdr(inputs, targets):
    return -si_snr(inputs, targets)

def l2_norm(s1, s2):
    norm = torch.sum(s1*s2, -1, keepdim=True)
    return norm

def si_snr(s1, s2, eps=1e-8):
    s1_s2_norm = l2_norm(s1, s2)
    s2_s2_norm = l2_norm(s2, s2)
    s_target =  s1_s2_norm/(s2_s2_norm+eps)*s2
    e_nosie = s1 - s_target
    target_norm = l2_norm(s_target, s_target)
    noise_norm = l2_norm(e_nosie, e_nosie)
    snr = 10*torch.log10((target_norm)/(noise_norm+eps)+eps)
    return torch.mean(snr)

def loss_phase_sensitive_spectral_approximation(enhance, target, mixture):
    """
    - Erdogan, Hakan, et al. "Phase-sensitive and recognition-boosted speech separation using deep recurrent neural networks."
    2015 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP). IEEE, 2015.

        Despite the use of the MSA/PSA objective function,
        it is still desirable to have the output of the network be a mask,
        since then the entire dynamic range of the data does not have to be covered by the output of the network.
        It is convenient in this context to truncate a to between 0 and 1, to fit the range of a sigmoid unit.
        In addition, we conjecture that mask prediction would also avoid global variance problems reported in [10].

        [10] “Global variance equalization for improving deep neural network based speech enhancement,” in Proc. of ICASSP,
        Florence, Italy, 2014.
    """
    eps = nn.Parameter(data=torch.ones((1, ), dtype=torch.float32)*1e-9, requires_grad=False).to(enhance.device)

    angle_mixture = torch.tanh(mixture[..., 1] / (mixture[..., 0]+eps))
    angle_target = torch.tanh(target[..., 1] / (target[..., 0]+eps))

    amplitude_enhance = torch.sqrt(enhance[..., 1]**2 + enhance[..., 0]**2)
    amplitude_target = torch.sqrt(target[..., 1]**2 + target[..., 0]**2)

    loss = amplitude_enhance - amplitude_target*torch.cos(angle_target-angle_mixture)
    loss = torch.mean(loss**2) # mse
    return loss

def UtterenceBaasedPermutationInvariantTraining(enhance, target: list, loss_function, mixture=None, return_comb=False):
    # O(S^2), S= the number of speaker
    assert enhance.shape == target.shape, f"enhance and target shape did not match...{enhance.shape}, {target.shape}"

    nspk_enhance = enhance.shape[1]
    nspk_target = target.shape[1]

    id_spks_enhance = list(range(nspk_enhance))
    id_spks_target = list(range(nspk_target))

    product_id_spks = product(id_spks_enhance, id_spks_target )
    loss_id_spks = torch.zeros(size=(nspk_enhance, nspk_target), dtype=torch.float32, requires_grad=False)
    with torch.no_grad():
        for i, (ienhance, itarget) in enumerate(product_id_spks):
            loss_id_spks[ienhance, itarget] = loss_function(enhance[:, ienhance, ...], target[:, itarget, ...]) if mixture is None else loss_function(enhance[:, ienhance, ...], target[:, itarget, ...], mixture)

        combinations_id_spks_enhance = permutations(id_spks_enhance)

        combination = None
        loss_min = 1e9
        for ienhance in combinations_id_spks_enhance:
            loss = 0
            comb = []
            for itarget in id_spks_target:
                loss += loss_id_spks[ienhance[itarget], itarget]
                comb.append((ienhance[itarget], itarget))

            if loss_min > loss:
                combination = comb
                loss_min = loss

    loss = torch.Tensor(torch.zeros(size=(1, )))
    loss = loss.to(target.device)
    for ienhance, itarget in combination:
        if mixture is None:
            loss += loss_function(enhance[:, ienhance, ...], target[:, itarget, ...])
        else:
            loss += loss_function(enhance[:, ienhance, ...], target[:, itarget, ...])
    loss /= nspk_enhance
    if not return_comb:
        return loss
    else:
        return loss, combination


'''
Comparing single layer MLP with deep MLP (using TensorFlow)
'''

import numpy as np
import pickle
from math import sqrt
from scipy.optimize import minimize

# Do not change this
def initializeWeights(n_in,n_out):
    """
    # initializeWeights return the random weights for Neural Network given the
    # number of node in the input layer and output layer

    # Input:
    # n_in: number of nodes of the input layer
    # n_out: number of nodes of the output layer

    # Output:
    # W: matrix of random initial weights with size (n_out x (n_in + 1))"""
    epsilon = sqrt(6) / sqrt(n_in + n_out + 1);
    W = (np.random.rand(n_out, n_in + 1)*2* epsilon) - epsilon;
    return W

# Replace this with your sigmoid implementation
def sigmoid(z):
    return  1.0 / (1.0 + np.exp(-z))
# Replace this with your nnObjFunction implementation
def nnObjFunction(params, *args):
    n_input, n_hidden, n_class, training_data, training_label, lambdaval = args

    w1 = params[0:n_hidden * (n_input + 1)].reshape((n_hidden, (n_input + 1)))
    w2 = params[(n_hidden * (n_input + 1)):].reshape((n_class, (n_hidden + 1)))
    obj_val = 0

    # Your code here

    trainingData_size = training_data.shape[0]

    training_output = np.zeros((trainingData_size,n_class))

    for i in range(len(training_label)):
        training_output[i][int(training_label[i])] = 1

    nnOutputValues, hidden_output_withBias, hidden_output, input_matrix_withBias = nnOutput(w1,w2,training_data)

    t1 = np.multiply(training_output, np.log(nnOutputValues))
    t2 = np.multiply(np.subtract(1,training_output),np.log(np.subtract(1,nnOutputValues)))

    t3 = np.add(t1,t2)

    obj_val = np.divide(np.sum(t3),-1*trainingData_size)

    reg = np.sum(np.power(w1,2)) + np.sum(np.power(w2,2))

    # Value of error function
    obj_val = obj_val + np.divide(np.multiply(reg,lambdaval),2*trainingData_size)

    # Gradient matrix calculation
    delta_l = np.subtract(nnOutputValues,training_output)

    Z_j = hidden_output_withBias

    # delta_Jmatrix is a (output layer neurons * hidden layer neurons) size matrix
    delta_Jmatrix = np.dot(delta_l.T,Z_j)

    temp = np.add(delta_Jmatrix,np.multiply(lambdaval,w2))

    grad_w2 = np.divide(temp,trainingData_size)

    # We remove the last column of the hidden layer since we are moving the opposite direction, i.e, back propagation
    w2_mod = np.delete(w2,n_hidden,1)

    # Same reason as above
    Z_j = hidden_output

    t4 = np.subtract(1,Z_j)

    t5 = np.multiply(t4,Z_j)

    t6 = np.dot(delta_l,w2_mod)

    t7 = np.multiply(t5,t6)

    delta_Jmatrix = np.dot(t7.T,input_matrix_withBias)

    grad_w1 = np.divide(np.add(delta_Jmatrix,np.multiply(lambdaval,w1)),trainingData_size)

    obj_grad = np.concatenate((grad_w1.flatten(), grad_w2.flatten()),0)

    print(obj_val)
    return (obj_val, obj_grad)

# Replace this with your nnPredict implementation
def nnPredict(w1,w2,data):
    ones_matrix = np.ones((data.shape[0],1))

    input_matrix_withBias = np.concatenate((data,ones_matrix),1)

    hidden_output = np.dot(input_matrix_withBias,w1.T)

    hidden_output = sigmoid(hidden_output)

    hidden_output_withBias = np.concatenate((hidden_output,ones_matrix),1)

    final_output = np.dot(hidden_output_withBias,w2.T)

    final_output = sigmoid(final_output)

    labels = np.argmax(final_output,axis=1)

    # labels = np.zeros((data.shape[0],1))
    # for i in range(data.shape[0]):
    #     labels[i,0] = np.argmax(final_output[i])

    return labels

def nnOutput(w1,w2,data):

    ones_matrix = np.ones((data.shape[0],1))

    input_matrix_withBias = np.concatenate((data,ones_matrix),1)

    hidden_output = np.dot(input_matrix_withBias,w1.T)

    hidden_output = sigmoid(hidden_output)

    hidden_output_withBias = np.concatenate((hidden_output,ones_matrix),1)

    final_output = np.dot(hidden_output_withBias,w2.T)

    final_output = sigmoid(final_output)

    return final_output, hidden_output_withBias, hidden_output, input_matrix_withBias

# Do not change this
def preprocess():
    pickle_obj = pickle.load(file=open('face_all.pickle', 'rb'))
    features = pickle_obj['Features']
    labels = pickle_obj['Labels']
    train_x = features[0:21100] / 255
    valid_x = features[21100:23765] / 255
    test_x = features[23765:] / 255

    labels = labels[0]
    train_y = labels[0:21100]
    valid_y = labels[21100:23765]
    test_y = labels[23765:]
    return train_x, train_y, valid_x, valid_y, test_x, test_y

"""**************Neural Network Script Starts here********************************"""
train_data, train_label, validation_data, validation_label, test_data, test_label = preprocess()
#  Train Neural Network
# set the number of nodes in input unit (not including bias unit)
n_input = train_data.shape[1]
# set the number of nodes in hidden unit (not including bias unit)
n_hidden = 256
# set the number of nodes in output unit
n_class = 2

# initialize the weights into some random matrices
initial_w1 = initializeWeights(n_input, n_hidden);
initial_w2 = initializeWeights(n_hidden, n_class);
# unroll 2 weight matrices into single column vector
initialWeights = np.concatenate((initial_w1.flatten(), initial_w2.flatten()),0)
# set the regularization hyper-parameter
lambdaval = 10;
args = (n_input, n_hidden, n_class, train_data, train_label, lambdaval)

#Train Neural Network using fmin_cg or minimize from scipy,optimize module. Check documentation for a working example
opts = {'maxiter' :50}    # Preferred value.

nn_params = minimize(nnObjFunction, initialWeights, jac=True, args=args,method='CG', options=opts)
params = nn_params.get('x')
#Reshape nnParams from 1D vector into w1 and w2 matrices
w1 = params[0:n_hidden * (n_input + 1)].reshape( (n_hidden, (n_input + 1)))
w2 = params[(n_hidden * (n_input + 1)):].reshape((n_class, (n_hidden + 1)))

#Test the computed parameters
predicted_label = nnPredict(w1,w2,train_data)
#find the accuracy on Training Dataset
print('\n Training set Accuracy:' + str(100*np.mean((predicted_label == train_label).astype(float))) + '%')
predicted_label = nnPredict(w1,w2,validation_data)
#find the accuracy on Validation Dataset
print('\n Validation set Accuracy:' + str(100*np.mean((predicted_label == validation_label).astype(float))) + '%')
predicted_label = nnPredict(w1,w2,test_data)
#find the accuracy on Validation Dataset
print('\n Test set Accuracy:' +  str(100*np.mean((predicted_label == test_label).astype(float))) + '%')


import numpy as np
import cv2
from matplotlib import pyplot as plt

I = cv2.imread('/home/kanish/Desktop/image.png', cv2.IMREAD_GRAYSCALE)

_, It = cv2.threshold(I, 0., 255, cv2.THRESH_OTSU)
It = cv2.bitwise_not(It)
_, labels = cv2.connectedComponents(I)

result = np.zeros((I.shape[0], I.shape[1], 3), np.uint8)

for i in range(labels.min(), labels.max() + 1):
    mask = cv2.compare(labels, i, cv2.CMP_EQ)

    _, ctrs, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)

    result = cv2.drawContours(result, ctrs, -1, (0xFF, 0, 0))

cv2.imwrite('new_image.png', result)


from fastapi import status, FastAPI, Request
from fastapi.exceptions import RequestValidationError
import os
from common.enum import MessageEnum
from common.constant import const
import logging
from .response_wrapper import resp_err

import traceback

logger = logging.getLogger(const.LOGGER_API)

def biz_exception(app: FastAPI):
    # customize request validation error
    @app.exception_handler(RequestValidationError)
    async def val_exception_handler(req: Request, rve: RequestValidationError, code: int = status.HTTP_400_BAD_REQUEST):
        lst = []
        for error in rve.errors():
            lst.append('{}=>{}'.format('.'.join(error['loc']), error['msg']))
        return resp_err(code, ' , '.join(lst))

    # customize business error
    @app.exception_handler(BizException)
    async def biz_exception_handler(req: Request, exc: BizException):
        return resp_err(exc.code, exc.message, exc.ref)

    # system error
    @app.exception_handler(Exception)
    async def exception_handler(req: Request, exc: Exception):
        if isinstance(exc, BizException):
            return
        error_msg = traceback.format_exc()
        if os.getenv(const.MODE) != const.MODE_DEV:
            error_msg = error_msg.replace("\n", "\r")
        logger.error(error_msg)
        return resp_err(MessageEnum.BIZ_UNKNOWN_ERR.get_code(),
                        MessageEnum.BIZ_UNKNOWN_ERR.get_msg())

class BizException(Exception):
    def __init__(self,
                 code: int = MessageEnum.BIZ_DEFAULT_ERR.get_code(),
                 message: str = MessageEnum.BIZ_DEFAULT_ERR.get_msg(),
                 ref: list = None):
        self.code = code
        self.message = message
        self.ref = ref


import cv2
import numpy as np
import os
import time
import pickle

from face_detection import RetinaFace
path = '../data/29--Students_Schoolkids/'
# model = 'resnet50'
model = 'mobilenet0.25'
scale = '1'
name = 'retinaFace'
count = 0
CONFIDENCE = 0.1
if __name__ == "__main__":
    for fn in os.listdir(path):
        filename = fn
        raw_img = cv2.imread(os.path.join(path, filename))
        detector = RetinaFace()
        out_file = '../data'
        name = fn.split('.')
        name = name[0]
        out_file = os.path.join(out_file, name.replace('jpg', 'txt'))
        t0 = time.time()
        print('start')
        faces = detector(raw_img)
        t1 = time.time()
        print(f'took {round(t1 - t0, 3)} to get {len(faces)} faces')
        # with open(out_file + '.txt', 'w') as f:
        #     # f.write("%s\n" % str(name))
        #     # f.write("%d\n" % len(faces))
        for box, landmarks, score in faces:
            box = box.astype(np.int)
            if score > CONFIDENCE:
                with open(out_file + '.txt', 'a') as f:
                    f.write("%s %g %d %d %d %d\n" % (str('face'), score, box[0], box[1], box[2] - box[0], box[3] - box[1]))
                    # f.write("%d %d %d %d %g\n" % (box[0], box[1], box[2] - box[0], box[3] - box[1], score))
                # cv2.rectangle(raw_img, (box[0], box[1]), (box[2], box[3]), color=(255, 0, 0), thickness=1)

        # while True:
        #     cv2.imshow('IMG', raw_img)
        #     if cv2.waitKey(1) & 0xFF == ord('q'):
        #         break


import unittest

from mock import ANY, Mock, patch

from captainhook import pre_commit

class TestMain(unittest.TestCase):

    def setUp(self):
        self.get_files_patch = patch('captainhook.pre_commit.get_files')
        get_files = self.get_files_patch.start()
        get_files.return_value = ['file_one']

        self.hook_config_patch = patch('captainhook.pre_commit.HookConfig')
        self.HookConfig = self.hook_config_patch.start()
        self.HookConfig().is_enabled.return_value = True
        self.HookConfig().arguments.return_value = ''

        self.testmod = Mock(spec=['run'])
        self.testmod.run.return_value = None
        self.checks_patch = patch('captainhook.pre_commit.checks')
        checks = self.checks_patch.start()
        checks.return_value = [("testmod", self.testmod)]

        self.mkdtemp_patch = patch('captainhook.pre_commit.tempfile.mkdtemp')
        self.mkdtemp_patch.start().return_value = '/tmp/dir'

        self.shutil_rm_patch = patch('captainhook.pre_commit.shutil.rmtree')
        self.shutil_rm_patch.start()

    def tearDown(self):
        self.checks_patch.stop()
        self.hook_config_patch.stop()
        self.get_files_patch.stop()
        self.mkdtemp_patch.stop()
        self.shutil_rm_patch.stop()

    def test_calling_run_without_args(self):
        result = pre_commit.main()

        self.assertEquals(result, 0)
        self.testmod.run.assert_called_with(['/tmp/dir/file_one'], '/tmp/dir')

    def test_calling_run_with_args(self):
        self.HookConfig().arguments.return_value = 'yep'

        result = pre_commit.main()

        self.assertEquals(result, 0)
        self.testmod.run.assert_called_with(
            ['/tmp/dir/file_one'], '/tmp/dir', 'yep'
        )

    @patch('captainhook.pre_commit.os.path.isfile')
    @patch('captainhook.pre_commit.shutil.copy')
    def test_required_files(self, copy, isfile):
        self.testmod.REQUIRED_FILES = ['should_be_copied']
        isfile.return_value = True

        pre_commit.main()

        copy.assert_called_with('should_be_copied', ANY)

    @patch('captainhook.pre_commit.os.path.isfile')
    @patch('captainhook.pre_commit.shutil.copy')
    def test_required_files_only_copied_if_exist(self, copy, isfile):
        self.testmod.REQUIRED_FILES = ['should_be_copied']
        isfile.return_value = False

        pre_commit.main()

        self.assertEquals(0, copy.call_count)


# coding: utf-8
from mock import mock

from django.contrib.auth.models import User
from django.conf import settings
from rest_framework import status

class AuthHelperMixin(object):

    def setUp(self):
        super(AuthHelperMixin, self).setUp()
        self.requests_patcher = mock.patch('sw_rest_auth.permissions.requests')
        self.requests_mock = self.requests_patcher.start()

    def tearDown(self):
        super(AuthHelperMixin, self).tearDown()
        self.requests_mock.stop()

    def assertPermChecked(self, user, perm):
        self.assertTrue(self.requests_mock.get.called)
        call_args, call_kwargs = self.requests_mock.get.call_args
        self.assertEqual(settings.AUTH_SERVICE_CHECK_PERM_URL, call_args[0])
        self.assertEqual(user.username, call_kwargs['params']['user'])
        self.assertEqual(perm, call_kwargs['params']['perm'])

    def force_permission(self, user=None, perm=None, perm_list=None):
        if not perm_list and perm:
            perm_list = [perm]

        def side_effect(url, headers, params, verify):
            response_mock = mock.Mock()
            if params['user'] == user.username and params['perm'] in perm_list:
                response_mock.status_code = status.HTTP_200_OK
            else:
                response_mock.status_code = status.HTTP_400_BAD_REQUEST
            return response_mock

        if user and perm_list:
            self.requests_mock.get.side_effect = side_effect
        else:
            self.requests_mock.get.side_effect = None

class AuthTestCaseMixin(AuthHelperMixin):
    url = None
    perm = None
    method_name = 'post'

    def get_method_names(self):
        if self.get_perm_map():
            return self.get_perm_map().keys()
        else:
            return [self.method_name]

    def get_perm_map(self):
        if self.perm:
            return {self.method_name: self.perm}
        else:
            return None

    def setUp(self):
        super(AuthTestCaseMixin, self).setUp()
        self.client.force_authenticate(self.get_user())
        if self.get_perm_map():
            self.force_permission(self.get_user(), perm_list=self.get_perm_map().values())

    def test_not_auth(self):
        self.client.force_authenticate()         # unset authentication
        for method_name in self.get_method_names():
            response = getattr(self.client, method_name)(self.url)
            self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)

    def test_forbidden(self):
        if not self.get_perm_map():
            return

        self.client.force_authenticate(user=self.get_user())
        self.force_permission()     # unset permission
        for method_name, perm_code in self.get_perm_map().items():
            response = getattr(self.client, method_name)(self.url)
            self.assertPermChecked(self.get_user(), perm_code)
            self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)

    def test_perm(self):
        if not self.get_perm_map():
            return

        self.client.force_authenticate(self.get_user())
        for method_name, perm_code in self.get_perm_map().items():
            self.force_permission(self.get_user(), perm=perm_code)
            response = getattr(self.client, method_name)(self.url)
            self.assertPermChecked(self.get_user(), perm_code)
            self.assertNotEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)
            self.assertNotEqual(response.status_code, status.HTTP_403_FORBIDDEN)

    def get_user(self):
        if not hasattr(self, 'user') or not self.user:
            self.user = User.objects.create(username='tester')
        return self.user


from time import sleep
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import argparse
from msedge.selenium_tools import Edge, EdgeOptions
import pandas as pd
import platform
import datetime
import pandas as pd
import multiprocessing as mp
from functools import partial
import sys
from glob import glob

class Tweet:
    """Wrapper class to contain scraped tweets data"""

    def __init__(self, username, timestamp, text, likes, retweets, replies, url):
        self.username = username
        self.timestamp = timestamp
        self.text = text
        self.likes = likes
        self.retweets = retweets
        self.url = url
        self.replies = replies

def get_data(card):
    """Extract data from tweet card"""
    try:
        handle = card.find_element_by_xpath(
            './/span[contains(text(), "@")]').text
    except:
        return

    try:
        postdate = card.find_element_by_xpath(
            './/time').get_attribute('datetime')
    except:
        return

    try:
        comment = card.find_element_by_xpath('.//div[2]/div[2]/div[1]').text
    except:
        comment = ""

    try:
        responding = card.find_element_by_xpath('.//div[2]/div[2]/div[2]').text
    except:
        responding = ""

    text = comment + responding

    try:
        reply_cnt = card.find_element_by_xpath(
            './/div[@data-testid="reply"]').text
    except:
        reply_cnt = 0

    try:
        retweet_cnt = card.find_element_by_xpath(
            './/div[@data-testid="retweet"]').text
    except:
        retweet_cnt = 0

    try:
        like_cnt = card.find_element_by_xpath(
            './/div[@data-testid="like"]').text
    except:
        like_cnt = 0

        # handle promoted tweets
    try:
        promoted = card.find_element_by_xpath(
            './/div[2]/div[2]/[last()]//span').text == "Promoted"
    except:
        promoted = False
    if promoted:
        return
    # tweet url
    try:
        element = card.find_element_by_xpath(
            './/a[contains(@href, "/status/")]')
        tweet_url = element.get_attribute('href')
    except:
        return
    like_cnt = 0 if not like_cnt else like_cnt
    retweet_cnt = 0 if not retweet_cnt else retweet_cnt
    reply_cnt = 0 if not reply_cnt else reply_cnt

    return Tweet(
        handle, postdate, text, like_cnt, retweet_cnt, reply_cnt, tweet_url
    )

def log_search_page(driver, start_date, end_date, lang, display_type, words):
    """Builds and searches for a query between start_date and end_date"""

    if words != None:
        words = str(words).split("//")
        words = "%20".join(words) + "%20"
    else:
        words = ""

    if lang != None:
        lang = 'lang%3A'+lang
    else:
        lang = ""

    end_date = "until%3A"+end_date+"%20"
    start_date = "since%3A"+start_date+"%20"

    query = f"https://twitter.com/search?q={words}"\
        f"{end_date}{start_date}{lang}&src=typed_query"
    driver.get(query)

    sleep(1)

    # navigate to historical 'Top' or 'Latest' tab
    try:
        driver.find_element_by_link_text(display_type).click()
    except:
        print("Latest Button doesnt exist.")

def init_driver(navig, headless, proxy):
    # create instance of web driver
    # path to the chromdrive.exe
    if navig == "chrome":
        browser_path = ''
        if platform.system() == 'Windows':
            print('Detected OS : Windows')
            browser_path = './drivers/chromedriver_win.exe'
        elif platform.system() == 'Linux':
            print('Detected OS : Linux')
            browser_path = './drivers/chromedriver_linux'
        elif platform.system() == 'Darwin':
            print('Detected OS : Mac')
            browser_path = './drivers/chromedriver_mac'
        else:
            raise OSError('Unknown OS Type')
        options = Options()
        if headless == True:
            options.headless = True
        else:
            options.headless = False
        options.add_argument('--disable-gpu')
        options.add_argument('log-level=3')
        if proxy != None:
            options.add_argument('--proxy-server=%s' % proxy)
        prefs = {"profile.managed_default_content_settings.images": 2}
        options.add_experimental_option("prefs", prefs)
        driver = webdriver.Chrome(
            options=options, executable_path=browser_path)
        driver.set_page_load_timeout(100)
        return driver
    elif navig == "edge":
        browser_path = 'drivers/msedgedriver.exe'
        options = EdgeOptions()
        if proxy != None:
            options.add_argument('--proxy-server=%s' % proxy)
        if headless == True:
            options.headless = True
            options.use_chromium = False
        else:
            options.headless = False
            options.use_chromium = True
        options.add_argument('log-level=3')
        driver = Edge(options=options, executable_path=browser_path)
        return driver

def keep_scroling(driver):
    """Keeps scrolling the Twitter feed and records tweets"""
    data = []
    scrolling = True
    last_position = driver.execute_script("return window.pageYOffset;")
    while scrolling:
        # get the card of tweets
        page_cards = driver.find_elements_by_xpath(
            '//div[@data-testid="tweet"]'
        )
        for card in page_cards:
            tweet = get_data(card)
            if tweet:
                data.append(tweet)
                last_date = tweet.timestamp
                print("Tweet made at: " + str(last_date)+" is found.")
        scroll_attempt = 0
        while True:
            driver.execute_script(
                'window.scrollTo(0, document.body.scrollHeight);')
            sleep(1)
            curr_position = driver.execute_script("return window.pageYOffset;")
            if last_position == curr_position:
                scroll_attempt += 1
                # end of scroll region
                if scroll_attempt > 1:
                    scrolling = False
                    break
                else:
                    sleep(1)  # attempt another scroll
            else:
                last_position = curr_position
                break
    return data

def make_output_path(words, init_date, max_date):
    """Builds an output path to save the tweets for given dates"""
    first_term = words.split("//")[0]
    first_date = str(init_date).split(" ")[0]
    first_end_date = str(max_date).split(" ")[0]
    return f"tweeter_output/{first_term}_{first_date}_{first_end_date}.csv"

def scrap(
    data_tuple,
    words,
    days_between,
    lang,
    display_type,
    driver,
    headless,
    proxy
):
    """
    scrap data from twitter using requests, starting from start_date until max_date. The bot make a search between each start_date and end_date
    (days_between) until it reaches the max_date.

    return:
    data : df containing all tweets scraped with the associated features.
    save a csv file containing all tweets scraped with the associated features.
    """
    start_date, end_date = data_tuple
    # initiate the driver
    driver = init_driver(driver, headless, proxy)

    # log search page between start_date and end_date
    log_search_page(driver=driver, words=words, start_date=datetime.datetime.strftime(start_date, '%Y-%m-%d'), end_date=datetime.datetime.strftime(
        end_date, '%Y-%m-%d'), lang=lang, display_type=display_type
    )

    print(f"looking for tweets between {start_date} and {end_date}...")
    count_failed = 0
    data = keep_scroling(driver)
    # retrying in case of timeouts of disconnections (up to 3 times)
    while len(data) == 0 and count_failed < 2:
        driver.close()
        driver = init_driver(driver, headless, proxy)
        log_search_page(driver=driver, words=words, start_date=datetime.datetime.strftime(start_date, '%Y-%m-%d'), end_date=datetime.datetime.strftime(
            end_date, '%Y-%m-%d'), lang=lang, display_type=display_type
        )
        data = keep_scroling(driver)
        count_failed += 1
    driver.close()
    return data

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Scrap tweets.')

    parser.add_argument('--words', type=str,
                        help='Queries. they should be devided by "//" : Cat//Dog.', default=None)
    parser.add_argument('--max_date', type=str,
                        help='Max date for search query. example : %%Y-%%m-%%d.', required=True)
    parser.add_argument('--start_date', type=str,
                        help='Start date for search query. example : %%Y-%%m-%%d.', required=True)
    parser.add_argument('--interval', type=int,
                        help='Interval days between each start date and end date for search queries. example : 5.', default=1)
    parser.add_argument('--navig', type=str,
                        help='Navigator to use : chrome or edge.', default="chrome")
    parser.add_argument('--lang', type=str,
                        help='Tweets language. example : "en" for english and "fr" for french.', default="en")
    parser.add_argument('--headless', type=bool,
                        help='Headless webdrives or not. True or False', default=True)
    parser.add_argument('--display_type', type=str,
                        help='Display type of twitter page : Latest or Top', default="Top")
    parser.add_argument('--proxy', type=str,
                        help='Proxy server', default=None)
    parser.add_argument('--custom_column_name', type=str,
                        help='A column to include for the words query', default="airline")

    args = parser.parse_args()

    words = args.words
    max_date = args.max_date
    start_date = args.start_date
    interval = args.interval
    navig = args.navig
    lang = args.lang
    driver = args.navig
    headless = args.headless
    display_type = args.display_type
    proxy = args.proxy
    custom_column_name = args.custom_column_name

    # monitor time
    start = datetime.datetime.now()
    # creating list of requirements
    start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d')
    end_date = datetime.datetime.strptime(max_date, '%Y-%m-%d')
    start_dates = [start_date]
    while start_dates[-1] + datetime.timedelta(days=interval) < end_date:
        start_dates.append(start_dates[-1] +
                           datetime.timedelta(days=interval))
    end_dates = [start_dates[i]
                 for i in range(1, len(start_dates))] + [end_date]
    # launching one worker thread per interval of time to scrape
    pool = mp.Pool(mp.cpu_count())
    try:
        all_tweets = pool.map(
            partial(
                scrap,
                words=words,
                days_between=interval,
                lang=lang,
                display_type=display_type,
                driver=driver,
                headless=headless,
                proxy=proxy,
            ),
            zip(start_dates, end_dates)
        )
        # merge and save
        merged_tweets = [
            t for interval_tweets in all_tweets for t in interval_tweets
        ]
        # save file
        df = pd.DataFrame([
            {**vars(tweet), **{custom_column_name: "_".join(words.split("//"))}}
            for tweet in merged_tweets
        ])
        df.to_csv(make_output_path(words, start_date, end_date), index=False)
        end = datetime.datetime.now()
        print(f"Time for current execution: {end - start}")
        print(f"Found {len(df.index)} tweets.")
    except KeyboardInterrupt:
        pool.terminate()
        pool.join()
        sys.exit()


import json
import time
import pymongo
import threading

from .TwitchWebsocket.TwitchWebsocket import TwitchWebsocket
from .FlushPrint import ptf

ws = None
statsDict = {}
statsLock = None
statsThread = None
colRewards = None

# True if user is a mod or the broadcaster
def CheckPrivMod(tags):
    return (tags["mod"] == "1" or
            tags["user-id"] == tags["room-id"])

# True if user is a sub, mod, or the broadcaster
def CheckPrivSub(tags):
    return (CheckPrivMod(tags) or
            tags["subscriber"] == "1")

def GetSyntax(user, syntax):
    return f"[{user}]: The syntax for that command is: {syntax}"

# Record that the given user has redeemed the given reward
def RedeemReward(user, rewardId):
    result = colRewards.find_one({ "user" : user })

    rewards = {}
    if result == None:
        rewards[rewardId] = 1
        userObj = {
            "user" : user,
            "rewards" : json.dumps(rewards)
        }
        colRewards.insert_one(userObj)
    else:
        rewards = json.loads(result["rewards"])

        if rewardId in rewards:
            rewards[rewardId] += 1
        else:
            rewards[rewardId] = 1

        colRewards.update_one(
            { "user" : user },
            { "$set" : { "rewards" : json.dumps(rewards) } }
        )

# Return true if the given user has redeemed the given reward and decrement
def CheckRemoveReward(user, rewardId):
    result = colRewards.find_one({ "user" : user })

    if result == None:
        return False

    rewards = json.loads(result["rewards"])

    if rewardId in rewards:
        if rewards[rewardId] == 1:
            del rewards[rewardId]
        else:
            rewards[rewardId] -= 1

        if len(rewards) == 0:
            colRewards.delete_one({ "user" : user })
        else:
            colRewards.update_one(
                { "user" : user },
                { "$set" : { "rewards" : json.dumps(rewards) } }
            )

        return True

    return False

# Log info for an incoming message
def LogReceived(type, user, message, tags, recordUsage=False):
    ptf(f"Received [{type}] from [{user}]: {message}", time=True)
    ptf(f"With tags: {tags}")

    if (recordUsage):
        token = message.lower().split(" ")[0]
        RecordUsage(token, user)

# Send a message to twitch chat and log
def SendMessage(response, type="PRIVMSG", user=None):
    global ws
    userStr = "" if user == None else f" to [{user}]"

    if response == None:
        ptf(f"No [{type}] message sent{userStr}\n", time=True)
        return

    if type == "WHISPER":
        ws.send_whisper(user, response)
    else:
        ws.send_message(response)

    ptf(f"Sent [{type}]{userStr}: {response}\n", time=True)

# Increment command usage of non-dev users
def RecordUsage(command, user):
    global statsDict
    global statsLock

    with statsLock:
        if command in statsDict:
            statsDict[command] += 1
        else:
            statsDict[command] = 1

# Dump current usage stats to file
def StoreUsageAsync():
    global statsDict
    global statsLock

    while True:
        # Every 10 minutes
        time.sleep(60 * 10)

        # No stats to dump
        if not statsDict:
            continue

        with open('UsageStats.json', 'r') as file:
            statsJson = json.load(file)

        with open('UsageStats.json', 'w') as file:
            with statsLock:
                for key in statsDict:
                    if key in statsJson:
                        statsJson[key] += statsDict[key]
                    else:
                        statsJson[key] = statsDict[key]

                statsDict = {}

            json.dump(statsJson, file, indent=4, sort_keys=True)

# Initialize util fields
def InitializeUtils(socket, chan, mongoClient):
    global ws
    global statsLock
    global statsThread
    global colRewards

    ws = socket
    colRewards = mongoClient.purebotdb[chan + "Rewards"]
    colRewards.create_index([("user", pymongo.ASCENDING)])

    with open('UsageStats.json', 'a+') as file:
        try:
            file.seek(0)
            json.load(file)
        except:
            file.truncate(0)
            file.write("{}\n")

    statsLock = threading.Lock()
    statsThread = threading.Thread(target=StoreUsageAsync)
    statsThread.start()


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

class DataVisualization(object):
	def __init__(self, data_frame) :
		self.data_frame = data_frame

	def view_histogram_by_column(self, column, title):
		try:
			sns.set(style='whitegrid')
			f, ax = plt.subplots(1,1, figsize=(6,4))
			ax = sns.histplot(self.data_frame[column], kde = True, color = 'c')
			plt.title(title, fontsize = 18, fontweight="bold")
			plt.xlabel(column.capitalize(), fontsize=14)
			plt.ylabel("Densidad", fontsize=14)
			plt.show();
		except KeyError:
			print("Problema para visualizar gráfico")

	def count_by_column(self, column):
		return self.data_frame.groupby(column).size()

	def view_distribution(self, hue, features):
		""" Return None
		Permite mostrar unos histogramas para ver la distribución de la categoría (hue) respecto de las variables indicadas en features
		"""
		sns.pairplot(self.data_frame, hue=hue, height=4, vars=features, kind='scatter');

	def view_correlation(self):
		plt.figure(figsize=(10, 10))
		sns.heatmap(self.data_frame.corr(), annot=True, cmap='Blues_r')
		plt.title("Correlación de características", fontsize=18, fontweight="bold")
		plt.show()

	def view_outliers(self, x, y):
  		plt.figure(figsize=(5,5))
  		sns.boxplot(x=x.lower(), y = y.lower(), data=self.data_frame)
  		plt.title("Outliers {}".format(y.capitalize()), fontsize=18, fontweight="bold")
  		plt.xlabel(x.capitalize(), fontsize=14)
  		plt.ylabel(y.capitalize(), fontsize=14)
  		plt.show()


import numpy as np
import matplotlib.pyplot as plt

def f(x, y):
    return x - y

def euler_method(f, x0, y0, h, num_steps):
    x_values = [x0]
    y_values = [y0]

    for _ in range(num_steps):
        x_next = x_values[-1] + h
        y_next = y_values[-1] + h * f(x_values[-1], y_values[-1])

        x_values.append(x_next)
        y_values.append(y_next)

    return x_values, y_values

x0 = 1
y0 = 1

h = 0.1
num_steps = 100

x_values, y_values = euler_method(f, x0, y0, h, num_steps)

plt.figure(figsize=(8, 6))
plt.plot(x_values, y_values, label="Euler's Method")
plt.xlabel('x')
plt.ylabel('y')
plt.title("Euler's Method for dy/dx = x - y")
plt.legend()
plt.grid(True)
plt.show()

Correct_Math 312_Group Delta/Commented Code/example_code 2_commented .py


# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from web_news.misc.spiderredis import SpiderRedis
from web_news.items import SpiderItem
from scrapy.loader import ItemLoader

class BjdSpider(SpiderRedis):
    name = 'bjd'
    allowed_domains = ['www.bjd.com.cn']
    start_urls = ['http://www.bjd.com.cn/']
    website = u'京报网'

    rules = (
        Rule(LinkExtractor(allow=r't(\d+)_(\d+)'), callback='parse_item', follow=False),
        Rule(LinkExtractor(allow=r'bjd.com.cn'), follow=True),
    )
    def parse_item(self, response):
        l = ItemLoader(item=SpiderItem(), response=response)
        try:
            date_source_author = response.xpath('//div[@class="info"]/span/text()').extract()
            l.add_value('title', response.xpath('//title/text()').extract_first() or '')
            l.add_value('date', date_source_author[0] if len(date_source_author)>0 else '1970-01-01 00:00:00')
            l.add_value('source', date_source_author[1] if len(date_source_author)>1 else '')
            l.add_value('content', ''.join(response.xpath('//div[@class="TRS_Editor"]/descendant-or-self::text()').extract()))
        except Exception as e:
            self.logger.error('error url: %s error msg: %s' % (response.url, e))
            l = ItemLoader(item=SpiderItem(), response=response)
            l.add_value('title', '')
            l.add_value('date', '1970-01-01 00:00:00')
            l.add_value('source', '')
            l.add_value('content', '')
            pass
        finally:
            l.add_value('url', response.url)
            l.add_value('collection_name', self.name)
            l.add_value('website', self.website)
            return l.load_item()


import unittest
from typing import List

'''
build on top of leetcode 84
cite from https://leetcode.com/problems/maximal-rectangle/discuss/122456/Easiest-solution-build-on-top-of-leetcode84
'''
class Solution:
    def maximalRectangle(self, matrix: List[List[str]]) -> int:
        if(len(matrix) == 0 or len(matrix[0]) == 0):
            return 0
        heights = [0] * len(matrix[0])
        maxArea = -1
        def largestRectangleArea(heights):
            # 避开了对edge case
            heights.append(0)
            stack, res, size = [-1], 0, len(heights)
            for i in range(size):
                # 后一个index小于前一个index时
                while heights[i] < heights[stack[-1]]:
                    height = heights[stack.pop()]
                    width = i - stack[-1] - 1
                    res = max(res, height * width)
                stack.append(i)
            # heights.pop()
            return res

        for i in range(len(matrix)):
            for j in range(len(matrix[0])):
                if matrix[i][j] == '0':
                    heights[j] = 0
                else:
                    heights[j] += 1
            area = largestRectangleArea(heights)
            maxArea = max(maxArea, area)
        return maxArea


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import division
import browser
import argparse
import sys
from mutagen import File
from mutagen.mp3 import HeaderNotFoundError
import os

DEFAULT_FOLDER = './'

def update_progress(progress, total):
    percent = int(progress / total * 100)
    sys.stdout.write("\r%d/%d (%d%%)" % (progress, total, percent))
    sys.stdout.flush()

def init_command_parser():
    parser = argparse.ArgumentParser()
    parser = argparse.ArgumentParser(description='Audit a music library')
    parser.add_argument('--folder', '-f', help='Music library folder', default=DEFAULT_FOLDER, action=browser.ReadableDir)

    return parser.parse_args()

def main():
    arguments = init_command_parser()

    print("Looking for files...")

    fileBrowser = browser.Browser(arguments.folder)
    files = list(fileBrowser.find(['mp3']))

    number_of_files = len(files)
    print("Working on %d files" % number_of_files)
    tags_to_delete = ["media", "comment", "copyright", "encodedby", "organization"]
    failed_to_open = []
    for i, f in enumerate(files):
        update_progress(i + 1, number_of_files)
        needs_saving = False
        try:
            music_file = File(os.path.join(f[1], f[0]), easy=True)
        except HeaderNotFoundError:
            failed_to_open.append(os.path.join(f[1], f[0]))
            continue

        if "performer" not in music_file or (music_file["performer"] != music_file['artist']):
            music_file["performer"] = music_file['artist']
            needs_saving = True

        for tag in tags_to_delete:
            if tag in music_file:
                del music_file[tag]
                needs_saving = True

        if needs_saving is True:
            music_file.tags.save()

    print("")
    print("Done")
    print("")
    for f in failed_to_open:
        print(f)
    print("")

if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        print("Bye !")
        sys.exit(0)