import datetime

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.chrome.options import Options

chrome_opt = Options()      # keep it open
chrome_opt.add_experimental_option('detach', True)
# from get_flight_data import FlightSearch

go_port = 'HKG'     # assumed we can get this result from previous operation
dest_port = 'NRT'
# flight_info = FlightSearch(current_port=go_port, dest_port=dest_port)
now = datetime.datetime.now()

go_date = now + datetime.timedelta(days=3) #departure date
back_date = go_date + datetime.timedelta(days=3) #return date

month_dict = {1: "January", 2: "February", 3: "March", 4: "April", 5: "May", 6: "June",
              7: "July", 8: "August", 9: "September", 10: "October", 11: "November", 12: "December"}

path = Service('chromedriver.exe')

class buy_ticket:
    def __init__(self):
        self.go_buy()

    def go_buy(self):
        driver = webdriver.Chrome(service=path, chrome_options=chrome_opt)

        driver.get('https://www.cathaypacific.com/cx/en_HK/book-a-trip.html')

        start_port = driver.find_element(By.XPATH, '//*[@id="depart-label"]')  # departure airport choice
        start_port.send_keys(go_port)
        start_port.send_keys(Keys.ENTER)

        end_port = driver.find_element(By.XPATH, '//*[@id="destination-label"]')  # destination airport choice
        end_port.send_keys(dest_port)
        end_port.send_keys(Keys.ENTER)

        dep_date = driver.find_element(By.XPATH, '//*[@id="dpuxzabv"]/span[2]/span[3]')  #departure date choice
        dep_date.click()

        # first_month = driver.find_element(By.XPATH, '//*[@id="dp1668851740859"]/div/div[1]/table/caption/span[1]')  # calender front part month
        # second_month = driver.find_element(By.XPATH, '//*[@id="dp1668851740859"]/div/div[2]/table/caption/span[1]')  #calender last part month

        # if first_month.text == month_dict[ flight_info.date.month]:  # departure date in first month choice

        calendar_dep_day = driver.find_elements(By.CLASS_NAME, 'ui-state-default')[::-1] # inverse to make sure it's valid even when it go to second month
        departure_day = go_date.day
        for ele in calendar_dep_day:
            if ele.text == departure_day:
                try:
                    ele.click()
                except NoSuchElementException:
                    continue

        re_day = driver.find_element(By.XPATH, '//*[@id="dppdvs3"]/span[2]/span[3]')  # return date choice
        re_day.click()

        calendar_re_day = driver.find_elements(By.CLASS_NAME, 'ui-state-default')[::-1] # inverse to make sure it's valid even when it go to second month
        play_duration = datetime.timedelta(days=3)
        return_day = back_date.day
        for ele in calendar_re_day:
            if ele.text == return_day:
                try:
                    ele.click()
                except NoSuchElementException:
                    continue

        submit_btm = driver.find_element(By.XPATH, '//*[@id="book-trip-flight"]/div[1]/form/button')
        submit_btm.click()


import streamlit as st
import pickle
import numpy as np

def load_model():
    with open('saved_steps.pkl', 'rb') as file:
        data = pickle.load(file)
    return data

data = load_model()

regressor = data["model"]
le_country = data["le_country"]
le_education = data["le_education"]

def show_predict_page():
    st.title("Software Developer Salary Prediction")

    st.write("""### Please provide the following information for salary prediction""")

    countries = (
        "Australia", "Austria", "Brazil", "Canada",
        "Denmark", "France", "Germany", "India", "Israel",
        "Italy", "Netherlands", "Norway", "Poland", "Portugal",
        "Spain", "Sweden", "Switzerland",
        "United Kingdom of Great Britain and Northern Ireland",
        "United States of America", "Other"
    )

    education = (
        "Less than a Bachelors",
        "Bachelor’s degree",
        "Master’s degree",
        "Post grad",
    )

    country = st.selectbox("Country", countries)
    education = st.selectbox("Education Level", education)

    experience = st.slider("Years of Experience", 0, 50, 3)

    ok = st.button("Calculate Salary")
    if ok:
        X = np.array([[country, education, experience ]])
        X[:, 0] = le_country.transform(X[:,0])
        X[:, 1] = le_education.transform(X[:,1])
        X = X.astype(float)

        salary = regressor.predict(X)
        st.subheader(f"The estimated salary is ${salary[0]:.2f}")


from utils.node_utils import ListNode, list_to_link

# Stack, T = O(n), S = O(n)
def isPalindrome_1(head: ListNode) -> bool:
    stack = []
    slow = fast = head
    while fast is not None and fast.next is not None:
        stack.append(slow.val)
        slow = slow.next
        fast = fast.next.next

    node = slow
    central = True
    while node is not None:
        if len(stack) > 0 and stack[-1] == node.val:
            stack.pop()
            node = node.next
        elif central:
            central = False
            node = node.next
        else:
            return False

    if len(stack) == 0:
        return True

    return False

# Reverse linked lists, T = O(n), S = O(1)
def isPalindrome_2(head: ListNode) -> bool:
    if head is None:
        return True

    slow = fast = head
    pre = pre2 = None
    while fast is not None and fast.next is not None:
        if pre is None:
            pre = slow
        else:
            tmp = pre.next
            pre.next = pre2
            pre2 = pre
            pre = tmp

        slow = slow.next
        fast = fast.next.next

    if pre is not None:
        pre.next = pre2
    central_off = True
    while pre is not None:
        if slow is None:
            return False

        if pre.val != slow.val:
            if central_off:
                slow = slow.next
                central_off = False
            else:
                return False
        else:
            pre = pre.next
            slow = slow.next

    return True

if __name__ == '__main__':
    assert isPalindrome_1(list_to_link([])) is True
    assert isPalindrome_1(list_to_link([1, 2])) is False
    assert isPalindrome_1(list_to_link([1, 2, 2, 1])) is True
    assert isPalindrome_1(list_to_link([1, 3, 1])) is True
    assert isPalindrome_2(list_to_link([])) is True
    assert isPalindrome_2(list_to_link([1, 2])) is False
    assert isPalindrome_2(list_to_link([1, 2, 2, 1])) is True
    assert isPalindrome_2(list_to_link([1, 3, 1])) is True


# coding: utf-8

# In[1]:

# 安装结巴分词
# pip install jieba
import jieba
import pandas as pd
import numpy as np
from tensorflow.keras.layers import Dense,Input,Dropout,Embedding,LSTM,Bidirectional
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.models import Model
from sklearn.model_selection import train_test_split
import json
# pip install plot_model
from plot_model import plot_model

# In[2]:

# 批次大小
batch_size = 128
# 训练周期
epochs = 3
# 词向量长度
embedding_dims = 128
# cell数量
lstm_cell = 64

# In[3]:

########step1-数据预处理########
# 这个数据前半部分都是正样本，后半部分都是负样本
data = pd.read_csv('weibo_senti_100k.csv')
# 计算正样本数量
poslen = sum(data['label']==1)
# 计算负样本数量
neglen = sum(data['label']==0)
#定义分词函数，对传入的x进行分词
cw = lambda x: list(jieba.cut(x))
# apply传入一个函数，把cw函数应用到data['review']的每一行
# 把分词后的结果保存到data['words']中
data['words'] = data['review'].apply(cw)
# 计算一条数据最多有多少个词汇
max_length = max([len(x) for x in data['words']])
# 把data['words']中所有的list都变成字符串格式
texts = [' '.join(x) for x in data['words']]
# 实例化Tokenizer，设置字典中最大词汇数为30000
# Tokenizer会自动过滤掉一些符号比如：!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n
tokenizer = Tokenizer(num_words=30000)
# 传入我们的训练数据，建立词典，词的编号根据词频设定，频率越大，编号越小，
tokenizer.fit_on_texts(texts)
# 把词转换为编号，编号大于30000的词会被过滤掉
sequences = tokenizer.texts_to_sequences(texts)
# 把序列设定为max_length的长度，超过max_length的部分舍弃，不到max_length则补0
# padding='pre'在句子前面进行填充，padding='post'在句子后面进行填充
X = pad_sequences(sequences, maxlen=max_length, padding='pre')

########step2-保存tokenizer########
# 把token_config保存到json文件中，模型预测阶段可以使用
file = open('token_config.json','w',encoding='utf-8')
# 把tokenizer变成json数据
token_config = tokenizer.to_json()
# 保存json数据
json.dump(token_config, file)

########step3-定义标签切分数据########
# 定义标签
# 01为正样本，10为负样本
positive_labels = [[0, 1] for _ in range(poslen)]
negative_labels = [[1, 0] for _ in range(neglen)]
# 合并标签
Y = np.array(positive_labels + negative_labels)
# 切分数据集
x_train,x_test,y_train,y_test = train_test_split(X, Y, test_size=0.2)

########step4-搭建模型########
# 定义函数式模型
# 定义模型输入，shape-(batch, 202)
sequence_input = Input(shape=(max_length,))
# Embedding层，30000表示30000个词，每个词对应的向量为128维
embedding_layer = Embedding(input_dim=30000, output_dim=embedding_dims)
# embedded_sequences的shape-(batch, 202, 128)
embedded_sequences = embedding_layer(sequence_input)
# 双向LSTM
x = Bidirectional(LSTM(lstm_cell))(embedded_sequences)

# 全连接层
x = Dense(128, activation='relu')(x)
# Dropout层
x = Dropout(0.5)(x)
# 输出层
preds = Dense(2, activation='softmax')(x)
# 定义模型
model = Model(sequence_input, preds)
# 画图
plot_model(model)

# In[4]:

########step5-模型训练和保存########
# 定义代价函数，优化器
model.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['acc'])

# 训练模型
model.fit(x_train, y_train,
          batch_size=batch_size,
          epochs=epochs,
          validation_data=(x_test, y_test))

# 保存模型
model.save('lstm_model.h5')

程序数据整理/第16章-NLP任务项目实战/16-4 双向LSTM中文微博情感分类.py


# -*- coding: utf-8 -*-
# @Time    : 2022-07-13 11:08
# @Author  : young wang
# @FileName: distortion_fix.py
# @Software: PyCharm
import glob
import numpy as np
import cv2 as cv
from matplotlib.patches import Circle
from scipy import ndimage
import matplotlib.pyplot as plt
from tools.pre_proc import folder_creator,load_from_oct_file
from tools.pos_proc import oct_to_dicom
from os.path import join
from tools.proc import max_slice,mip_stack,binary_mask
import matplotlib

if __name__ == '__main__':

    matplotlib.rcParams.update(
        {
            'font.size': 13.5,
            'text.usetex': False,
            'font.family': 'sans-serif',
            'mathtext.fontset': 'stix',
        }
    )

    data = glob.glob('../data/2022.07.13_1mm(3dprint)/trial 5/*.oct')

    # Define the dimensions of checkerboard
    height, width = 4, 4
    checked_board = (height, width)
    criteria = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 1000, 0.0001)

    # Initialize enpty list to accumulate coordinates
    objpoints = []  # 3d world coordinates
    imgpoints = []  # 2d image coordinates

    objp = np.zeros((1,
                     checked_board[0] * checked_board[1], 3), np.float32)
    objp[0, :, :2] = np.mgrid[0:checked_board[0], 0:checked_board[1]].T.reshape(-1, 2)

    fig, ax = plt.subplots(1,len(data), figsize = (16, 9))

    for i in range(len(data)):

        data_ori = load_from_oct_file(data[i])

        index = max_slice(data_ori)

        # constructed maximum intensity projections from a stack with
        # certain thickness
        pad = 10
        mip_slice = mip_stack(data_ori, index, pad)

        p_factor = 0.65
        vmin, vmax = int(255 * p_factor), 255

        mask = binary_mask(mip_slice,vmin, vmax)
        mask = ndimage.median_filter(mask, size=3)
        mask = ndimage.gaussian_filter(mask, sigma=0.1)

        res = np.uint8(mask)
        ret, corners = cv.findChessboardCorners(res, checked_board,
                                                flags=cv.CALIB_CB_ADAPTIVE_THRESH +
                                                      cv.CALIB_CB_FAST_CHECK +
                                                      cv.CALIB_CB_NORMALIZE_IMAGE)
        if ret:

            # If found, add object points, image points (after refining them)
            objpoints.append(objp)

            corners2 = cv.cornerSubPix(res, corners, (13, 13), (-1, -1), criteria)
            imgpoints.append(corners2)

            cv.drawChessboardCorners(res, checked_board, corners2, ret)

            corners2 = np.squeeze(corners2)
            for corner in corners2:
                coord = (int(corner[0]), int(corner[1]))
                circ = Circle(coord, 5, edgecolor='r', fill=True, linewidth=1, facecolor='r')
                ax[i].add_patch(circ)

        ax[i].set_title('trial '+ str(i), size = 20)
        ax[i].set_axis_off()
        ax[i].imshow(res,'gray', vmin= vmin, vmax = vmax )
    plt.tight_layout()
    plt.show()

    h, w = res.shape[:2]
    ret, mtx, dist, rvecs, tvecs = cv.calibrateCamera(objpoints, imgpoints, res.shape[::-1], None, None)

    val_data = load_from_oct_file('../data/validation/2022-Jul-13_02.04.07_PM_Bin_Capture.oct')
    index = max_slice(val_data)

    # constructed maximum intensity projections from a stack with
    # certain thickness
    pad = 10
    val_slice = mip_stack(val_data, index, pad)

    # undistort
    newcameramtx, roi = cv.getOptimalNewCameraMatrix(mtx, dist, (w, h), 1, (w, h))
    mapx, mapy = cv.initUndistortRectifyMap(mtx, dist, None, newcameramtx, (w, h), 5)
    dst = cv.remap(val_slice, mapx, mapy, cv.INTER_LINEAR)

    fig,ax = plt.subplots(1,2, figsize = (16,9))
    ax[0].set_title('distorted validation image')
    ax[0].set_axis_off()
    ax[0].imshow(val_slice, 'gray', vmin=vmin, vmax=vmax)

    ax[1].set_title('undistorted validation image')
    ax[1].set_axis_off()
    ax[1].imshow(dst, 'gray', vmin=vmin, vmax=vmax)

    plt.tight_layout()
    plt.show()

    #create dicom stacks for comparison
    dicom_path = join('../', 'validation dicom')
    resolutionx, resolutiony, resolutionz = 0.026, 0.026, 0.030

    folder_creator(dicom_path)

    file_path = 'distorted'
    f_path = join(dicom_path,file_path)
    folder_creator(f_path)

    patient_info = {'PatientName': 'RESEARCH',
                    'PatientBirthDate': '19600507',
                    'PatientSex': 'F',
                    'PatientAge': '63Y',
                    'PatientID': '202207070001',
                    'SeriesDescription': file_path,
                    'StudyDescription': 'OCT 3D'}

    oct_to_dicom(val_data, resolutionx=resolutionx,
                 resolutiony=resolutiony,resolutionz = resolutionz,
                 dicom_folder=f_path,
                 **patient_info)
    #
    print('Done creating dicom stacks for distorted validation dataset')

    val_undis = np.zeros_like(val_data)
    for i in range(val_data.shape[-1]):
        val_undis[:,:,i] = cv.remap(val_data[:,:,i], mapx, mapy, cv.INTER_LINEAR)

    print('Done undistorting validation dataset')

    file_path = 'undistorted'
    f_path = join(dicom_path,file_path)
    folder_creator(f_path)

    patient_info = {'PatientName': 'RESEARCH',
                    'PatientBirthDate': '19600507',
                    'PatientSex': 'F',
                    'PatientAge': '63Y',
                    'PatientID': '202107070001',
                    'SeriesDescription': file_path,
                    'StudyDescription': 'OCT 3D'}

    oct_to_dicom(val_undis, resolutionx=resolutionx,
                 resolutiony=resolutiony,resolutionz = resolutionz,
                 dicom_folder=f_path,
                 **patient_info)
    #
    print('Done creating dicom stacks for undistorted validation dataset')


from __future__ import annotations

import os
import sys

# If python version is <  3.10 skip test
# glob on that version of Python doesnt have root_dir param
if sys.version_info < (3, 10):
    print("Skip test")
    sys.exit(0)

from dynaconf import Dynaconf

settings = Dynaconf(
    environments=True,
    lowercase_read=False,
    load_dotenv=True,
    root_path=os.path.dirname(os.path.abspath(__file__)),
    settings_files=["settings.toml", ".config/*.yaml"],
)

assert settings.LOG_LEVEL == "INFO"
assert settings.DATA_SOURCES == {"NAME": 1}
print(settings.LOG_LEVEL)
print(settings.DATA_SOURCES)


 #the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" This is a generic unit testing lib for
    Apache Warble (incubating) nodes.
"""

import plugins.basics
import plugins.tests
import datetime

def spit(t):
    # All done, spit out report
    print("TEST %s COMPLETED" % t.report.id)
    print()

    print("DEBUG:")
    print('-' * 80)
    for k, v in t.report._debug:
        print(datetime.datetime.fromtimestamp(k).strftime("%Y-%m-%d %H:%M:%S.%f"), v)
    print('-' * 80)
    print("TIMESTAMPS:")
    print('-' * 80)
    previous = None
    first = None
    for k, v in sorted(t.report.timeseries.items(), key = lambda x: x[1]):
        if not previous:
            previous = v
            first = v
        print("%-10s: +%3dms (%3dms)" % (k, ((v - previous) * 1000), (v - first) * 1000))
        previous = v
    print('-' * 80)

def uprint(t, params):
    print("Running test %s:" % t.report.id)
    print('-' * 80)
    for k, v in params.items():
        print("%-16s: %s" % (k, v))
    print('-' * 80)

def run(gc):
    gc['debug'] = True

    # TCP test
    params = {
        'host': 'www-us.apache.org',
        'port': 80
    }
    t = plugins.tests.tcp.test(gc)
    uprint(t, params)
    t.run(params)
    spit(t)

    # HTTP test
    params = {
        'host': 'www-eu.apache.org',
        'vhost': 'www.apache.org',
        'URI': '/',
        'port': 80
    }
    t = plugins.tests.http.test(gc)
    uprint(t, params)
    t.run(params)
    spit(t)

    # SMTP test
    params = {
        'host': 'mail-relay.apache.org',
        'type': 'smtp',
        'SSL': True,
        'port': 465
    }
    uprint(t, params)
    t = plugins.tests.smtp.test(gc)
    t.run(params)
    spit(t)


# -*- coding: utf-8 -*-
from pyspark.sql import SparkSession
from pyspark.ml.regression import LinearRegression
from pyspark.ml.feature import VectorAssembler
import numpy as np
import matplotlib.pyplot as plt

spark=SparkSession.builder.getOrCreate()
df=spark.read.csv('D:\PY\ML\m_share.csv',header=True)
datas=df.select(df.view.cast('double'),df.like.cast('double'),df.danmaku.cast('double'),df.reply.cast('double'),df.share.cast('double'))

#datas=df.select(df.view.cast('double'),df.like.cast('double'),df.danmaku.cast('double'),df.reply.cast('double'),df.share.cast('double'))

(trainingdata,testdata)=datas.randomSplit([0.7,0.3])

assembler=VectorAssembler(inputCols=["view","like","danmaku","reply"],outputCol="features")
output=assembler.transform(trainingdata)
label_features=output.select("features","share").toDF('features','label')
label_features.show(truncate=False)

lr=LinearRegression(maxIter=10,regParam=0.3,elasticNetParam=0.8)
lrModel=lr.fit(label_features)
print("Coefficients: %s" % str(lrModel.coefficients))
print("Intercept: %s" % str(lrModel.intercept))

trainingSummary = lrModel.summary
print("numIterations: %d" % trainingSummary.totalIterations)
print("objectiveHistory: %s" % str(trainingSummary.objectiveHistory))

trainingSummary.residuals.show()
print("RMSE: %f" % trainingSummary.rootMeanSquaredError)
print("r2: %f" % trainingSummary.r2)

output1=assembler.transform(testdata)
test_features=output1.select("features","share").toDF('features','label')

result=lrModel.transform(label_features).collect()
reallist=[]
prelist=[]
ccc=0
for it in result:
    if ccc==50:
        break
    reallist.append(it['label'])
    prelist.append(it['prediction'])
    ccc+=1
print(reallist)
print(prelist)

fig=plt.figure(figsize=(10,6))
plt.ylim(0,10000)
ax=fig.add_subplot(1,1,1)
ax.set_title('Test')
ax.set_xlabel('number')
ax.set_ylabel('video_score')
ticks=ax.set_xticks(np.arange(50))
ax.scatter(np.arange(50),prelist,label='Pred_Value')
ax.scatter(np.arange(50),reallist,label='Label_Value')
ax.legend(loc='best')
plt.savefig('D:\PY\GraphX\handle\png\graph2.png',dpi=400,bbox_inches='tight')


# coding=utf-8
from gensim.models import Word2Vec
from gensim.models.word2vec import LineSentence

from utils import config
from tools import cpu_cores

def train_word2vec(file_path=config.merged_seg_path):

    # 训练词向量
    model = Word2Vec(
        LineSentence(source=file_path),
        vector_size=config.embedding_dim,
        sg=1,
        workers=cpu_cores,
        window=5,
        min_count=5,
        epochs=config.word2vec_train_epochs,
    )

    return model


import fnmatch
import os
import shlex
import time

def _combine(*iterables):
    iterables = [iter(iterable) for iterable in iterables]
    size = len(iterables)
    while True:
        to_yield = [None] * size
        for idx, iterable in enumerate(iterables):
            if iterable is None:
                continue
            try:
                to_yield[idx] = iterable.next()
            except StopIteration:
                iterables[idx] = None
        if not [iterable for iterable in iterables if iterable is not None]:
            break
        yield tuple(to_yield)

class CommandLine(object):
    """Simple helper for executing subprocesses."""

    def __init__(self, executable, args, input=None, cwd=None):
        """Initialize the CommandLine object.

        :param executable: the name of the program to execute
        :param args: a list of arguments to pass to the executable
        :param input: string or file-like object containing any input data for
                      the program
        :param cwd: the working directory to change to before executing the
                    command
        """
        self.executable = executable
        self.arguments = [str(arg) for arg in args]
        self.input = input
        self.cwd = cwd
        if self.cwd:
            assert os.path.isdir(self.cwd)
        self.returncode = None

    if os.name == 'nt': # windows

        def execute(self, timeout=None):
            """Execute the command, and return a generator for iterating over
            the output written to the standard output and error streams.

            :param timeout: number of seconds before the external process
                            should be aborted (not supported on Windows)
            """
            args = [self.executable] + self.arguments
            for idx, arg in enumerate(args):
                if arg.find(' ') >= 0:
                    args[idx] = '"%s"' % arg
            #log.debug('Executing %s', args)

            if self.cwd:
                old_cwd = os.getcwd()
                os.chdir(self.cwd)

            import tempfile
            in_name = None
            if self.input:
                if isinstance(self.input, basestring):
                    in_file, in_name = tempfile.mkstemp(prefix='bitten_',
                                                        suffix='.pipe')
                    os.write(in_file, self.input)
                    os.close(in_file)
                    in_redirect = '< "%s" ' % in_name
                else:
                    in_redirect = '< "%s" ' % self.input.name
            else:
                in_redirect = ''

            out_file, out_name = tempfile.mkstemp(prefix='bitten_',
                                                  suffix='.pipe')
            os.close(out_file)
            err_file, err_name = tempfile.mkstemp(prefix='bitten_',
                                                  suffix='.pipe')
            os.close(err_file)

            try:
                cmd = '( %s ) > "%s" %s 2> "%s"' % (' '.join(args), out_name,
                                                    in_redirect, err_name)
                self.returncode = os.system(cmd)
                #log.debug('Exited with code %s', self.returncode)

                out_file = file(out_name, 'r')
                err_file = file(err_name, 'r')
                out_lines = out_file.readlines()
                err_lines = err_file.readlines()
                out_file.close()
                err_file.close()
            finally:
                if in_name:
                    os.unlink(in_name)
                if out_name:
                    os.unlink(out_name)
                if err_name:
                    os.unlink(err_name)
                if self.cwd:
                    os.chdir(old_cwd)

            for out_line, err_line in _combine(out_lines, err_lines):
                yield out_line and out_line.rstrip().replace('\x00', ''), \
                      err_line and err_line.rstrip().replace('\x00', '')

    else: # posix

        def execute(self, timeout=None):
            """Execute the command, and return a generator for iterating over
            the output written to the standard output and error streams.

            :param timeout: number of seconds before the external process
                            should be aborted (not supported on Windows)
            """
            import popen2, select
            if self.cwd:
                old_cwd = os.getcwd()
                os.chdir(self.cwd)

            #log.debug('Executing %s', [self.executable] + self.arguments)
            pipe = popen2.Popen3([self.executable] + self.arguments,
                                 capturestderr=True)
            if self.input:
                if isinstance(self.input, basestring):
                    in_data = self.input
                else:
                    in_data = self.input.read()
            else:
                pipe.tochild.close()
                in_data = ''

            out_data, err_data = [], []
            in_eof = out_eof = err_eof = False
            if not in_data:
                in_eof = True
            while not out_eof or not err_eof:
                readable = [pipe.fromchild] * (not out_eof) + \
                           [pipe.childerr] * (not err_eof)
                writable = [pipe.tochild] * (not in_eof)
                ready = select.select(readable, writable, [], timeout)
                if not (ready[0] or ready[1]):
                    raise TimeoutError('Command %s timed out' % self.executable)
                if pipe.tochild in ready[1]:
                    sent = os.write(pipe.tochild.fileno(), in_data)
                    in_data = in_data[sent:]
                    if not in_data:
                        pipe.tochild.close()
                        in_eof = True
                if pipe.fromchild in ready[0]:
                    data = os.read(pipe.fromchild.fileno(), 1024)
                    if data:
                        out_data.append(data)
                    else:
                        out_eof = True
                if pipe.childerr in ready[0]:
                    data = os.read(pipe.childerr.fileno(), 1024)
                    if data:
                        err_data.append(data)
                    else:
                        err_eof = True
                out_lines = self._extract_lines(out_data)
                err_lines = self._extract_lines(err_data)
                for out_line, err_line in _combine(out_lines, err_lines):
                    yield out_line, err_line
                #time.sleep(.1)
            self.returncode = os.WEXITSTATUS(pipe.wait())
            #log.debug('%s exited with code %s', self.executable,
            #          self.returncode)

            if self.cwd:
                os.chdir(old_cwd)

    def _extract_lines(self, data):
        extracted = []
        def _endswith_linesep(string):
            for linesep in ('\n', '\r\n', '\r'):
                if string.endswith(linesep):
                    return True
        buf = ''.join(data)
        lines = buf.splitlines(True)
        if len(lines) > 1:
            extracted += lines[:-1]
            if _endswith_linesep(lines[-1]):
                extracted.append(lines[-1])
                buf = ''
            else:
                buf = lines[-1]
        elif _endswith_linesep(buf):
            extracted.append(buf)
            buf = ''
        data[:] = [buf] * bool(buf)

        return [line.rstrip() for line in extracted]

def locate(fn):
    """Locates a binary on the path.

    Returns the fully-qualified path, or None.
    """
    if os.name == 'nt':
        fn += '.exe'

    for p in ["."] + os.environ['PATH'].split(os.pathsep):
        f = os.path.join(p, fn)
        if os.path.exists(f):
            return f
    return None


# Access log parser v1.0
# Author: Sergei Safarov <inbox@sergeysafarov.com>

import csv
import os
import re
import sys
import geoip2.database
import wget
import gzip
import shutil
import glob

from os import path
from user_agents import parse

########################################################################################################################
# Functions declarations
# Start
########################################################################################################################

def file_name(filename):
    return os.path.splitext(os.path.basename(filename))[0]

def file_size(filename):
    return os.path.getsize(filename)

########################################################################################################################
# Settings
########################################################################################################################
deafult_log_file = 'https://cti-developer-dropbox.s3.amazonaws.com/gobankingrates.com.access.log'

lang = 'en'
input_file = 'log/access.log'
geoip_database = 'GeoLite2-Country.mmdb'
geoip_db_source = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.mmdb.gz'

########################################################################################################################
# Script body
########################################################################################################################

if not path.exists(input_file):
    print('Access log file not found in local folder. Trying to download from default remote location ' + deafult_log_file)
    wgt_result = wget.download(deafult_log_file, 'log/')
    if wgt_result:
        print('Access log file downloaded from default remote location.')
        file_list = glob.glob ('log/*.log')
        if len(file_list) > 0 and path.exists(file_list[0]):
            input_file = file_list[0]
            print('Using downloaded file ' + input_file)

if not path.exists(input_file):
    sys.exit('Inboud log file not found. Stopped')

if_size = file_size(input_file)

if if_size == 0:
    sys.exit('Inboud log file is empty. Stopped')

if not path.exists(geoip_database):
    print('Geo IP database not found. Trying to download from ' + geoip_db_source)
    wgt_result = wget.download(geoip_db_source)
    if wgt_result:
        with gzip.open('GeoLite2-Country.mmdb.gz', 'rb') as f_in:
            with open('GeoLite2-Country.mmdb', 'wb') as f_out:
                print('Geo IP database downloaded. Extracting.')
                shutil.copyfileobj(f_in, f_out)
                print('Geo IP database extracted.')
        # Clean downloaded compressed db file
        if path.exists('GeoLite2-Country.mmdb.gz'):
            os.remove('GeoLite2-Country.mmdb.gz')

if not path.exists(geoip_database):
    print('Geo IP database not found. Countries will not be retrieved. ')
    print('Please obtain the database ' + geoip_database + ' file and put it into the script folder')
else:
    geo_ip_reader = geoip2.database.Reader(geoip_database)

output_file = file_name(input_file) + '.csv'

print('Parsing ' + input_file + ' [' + str(if_size) + ' bytes]')
print('Writing ' + output_file)

with open(input_file, encoding="UTF8") as fi, open(output_file, 'w', encoding="UTF8") as fo:
    writer = csv.writer(fo, quotechar='"', quoting=csv.QUOTE_NONNUMERIC)
    writer.writerow(['#', 'IP', 'Country', 'Browser', 'Device', 'Type', 'RAW LINE'])

    i = 1
    for line in fi:
        sys.stdout.write("Processing line: %d   \r" % (i))
        sys.stdout.flush()

        ip_address = re.findall(r'[0-9]+(?:\.[0-9]+){3}', line)[0]
        try:
            # No problems here in case of geo DB not found as we are inside try/except block
            location = geo_ip_reader.country(ip_address)

            if location and location.country and location.country.names:
                country = location.country.names[lang]
            else:
                country = 'N/A'
        except:
            country = 'N/A'

        try:
            line_parts = re.findall(r'\"(.+?)\"', line)
            ua_record = line_parts[len(line_parts) - 1]
            parsed_ua = parse(ua_record)

            browser = parsed_ua.browser.family
            device_family = parsed_ua.device.family

            if parsed_ua.is_mobile:
                device_type = 'Mobile'
            elif parsed_ua.is_tablet:
                device_type = 'Tablet'
            elif parsed_ua.is_pc:
                device_type = 'Desktop'
            elif parsed_ua.is_bot:
                device_type = 'Bot'
            else:
                device_type = 'Unknown'
        except:
            browser = 'N/A'
            device_family = 'N/A'
            device_type = 'N/A'

        writer.writerow([i, ip_address, country, browser, device_family, device_type, line])
        i += 1

print("Parsing finished. " + str(i - 1) + " lines processed.")


# -*- coding: utf-8 -*-
"""
Created on Wed Sep  6 20:39:35 2017

@author: 白家鹏
"""

from sklearn.linear_model import LinearRegression, RidgeCV
import numpy as np
import pandas as pd
from sklearn.cross_validation import train_test_split
from sklearn.metrics import mean_squared_error
from sklearn import neighbors
from sklearn import datasets
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.model_selection import cross_val_score

ForexData = pd.read_csv('AUDUSD_30days.csv')

# Get the features X and target/response y
X =ForexData.iloc[:, 2:]
y = ForexData.iloc[:, 1]

# Split data into train and test sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)

"""
OLS

"""
lm = LinearRegression()

lm.fit(X_train, y_train)

preds_ols = lm.predict(X_test)

print("OLS: {0}".format(mean_squared_error(y_test, preds_ols)/2))

"""
RIDGE

"""
alpha_range = 10.**np.arange(-2, 3)

rregcv = RidgeCV(normalize=True, scoring='neg_mean_squared_error', alphas=alpha_range)

rregcv.fit(X_train, y_train)

preds_ridge = rregcv.predict(X_test)
print("RIDGE: {0}".format(mean_squared_error(y_test, preds_ridge)/2))

"""
LassoCV

"""
from sklearn.linear_model import LassoCV, ElasticNetCV

lascv = LassoCV(normalize=True, tol=0.01)
lascv.fit(X_train, y_train)

preds_lassocv = lascv.predict(X_test)
print("LASSO: {0}".format(mean_squared_error(y_test, preds_lassocv)/2))
print("LASSO Lambda: {0}".format(lascv.alpha_))
"""
ElasticNetCV

"""
elascv = ElasticNetCV(normalize=True, tol=0.01)
elascv.fit(X_train, y_train)

preds_elascv = elascv.predict(X_test)
print("Elastic Net: {0}".format(mean_squared_error(y_test, preds_elascv)/2))
print("Elastic Net Lambda: {0}".format(elascv.alpha_))

"""
b

"""
m=245
#obtain features of LeBron_James and Kevin_Durant
row = X.iloc[m, :]
Test_predictions_LM = lm.predict(row)
Test_predictions_RISGE = rregcv.predict(row)
Test_predictions_LASSO = lascv.predict(row)
Test_predictions_ELA = elascv.predict(row)
Test_realValue=y.iloc[m]


bl_info = {
    "name": "Blender Command Port",
    "description": "Command Port implementation for blender. "
                   "www.github.com/p4vv37/blender_command_port"
                   "www.github.com/jeffhanna/blender_command_port",
    "author": "Pawel Kowalski, Jeff Hannna",
    "version": (1, 2, 1),
    "blender": (2, 80, 0),
    "location": "User preferences > Blender Command Port",
    "support": "COMMUNITY",
    "wiki_url": "www.github.com/p4vv37/blender_command_port",
    "category": "Development",
}

import bpy
from bpy.types import Panel

from .command_port import register as register_command_port
from .command_port import unregister as unregister_command_port
from .command_port import CommandPortOperator

class OpenCommandPortOperator(bpy.types.Operator):
    bl_idname = "wm.open_command_port"
    bl_label = "Open Port"

    # noinspection PyMethodMayBeStatic,PyUnusedLocal
    def execute(self, context):
        bpy.ops.object.command_port('INVOKE_DEFAULT')
        return {'FINISHED'}

class CloseCommandPortOperator(bpy.types.Operator):
    bl_idname = "wm.close_command_port"
    bl_label = "Close Port"

    # noinspection PyMethodMayBeStatic
    def execute(self, context):
        try:
            if not CommandPortOperator.instance.is_alive():
                print("Port is not running")
                return False
            CommandPortOperator.instance.do_run = False
            print("Command port closed")
        except NameError:
            print("Port is not running. It was never initialized.")
        return {'FINISHED'}

class BLENDERCOMMANDPORT1_PT_Panel(Panel):
    bl_label = 'Blender Command Port'

    # 2.8 uses PREFERENCES and system. There is no need to attempt to context
    # switch between Blender 2.7 and 2.8 here as the 'blender' version metadata
    # at the top of this file limits it to Blender 2.80 or later.

    bl_space_type = 'PREFERENCES'
    bl_context = 'system'
    bl_region_type = 'WINDOW'

    def draw(self, context):
        layout = self.layout
        window_manager = context.window_manager
        layout.prop(window_manager, 'bcp_queue_size')
        layout.prop(window_manager, 'bcp_timeout')
        layout.prop(window_manager, 'bcp_port')
        layout.prop(window_manager, 'bcp_buffersize')
        layout.prop(window_manager, 'bcp_max_connections')
        layout.prop(window_manager, 'bcp_return_result')
        layout.prop(window_manager, 'bcp_result_as_json')
        layout.prop(window_manager, 'bcp_redirect_output')
        layout.prop(window_manager, 'bcp_share_environ')
        row = layout.row()
        try:
            port_running = CommandPortOperator.instance.is_alive()
        except AttributeError:
            port_running = False

        if port_running:
            row.operator("wm.close_command_port")
        else:
            row.operator("wm.open_command_port")

def register():
    bpy.utils.register_class(OpenCommandPortOperator)
    bpy.utils.register_class(CloseCommandPortOperator)
    register_command_port()
    bpy.utils.register_class(BLENDERCOMMANDPORT1_PT_Panel)

def unregister():
    bpy.utils.unregister_class(OpenCommandPortOperator)
    bpy.utils.unregister_class(CloseCommandPortOperator)
    unregister_command_port()
    bpy.utils.unregister_class(BLENDERCOMMANDPORT1_PT_Panel)


# ARQUIVO DE TESTE DE IMPLEMENTAÇÃO
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.keys import Keys

def clima(usuario):
    options = Options()
    options.add_argument('--headless')
    nav = webdriver.Chrome(options=options)
    nav.get('http:www.google.com')
    caminho = '/html/body/div[1]/div[3]/form/div[1]/div[1]/div[1]/div/div[2]/input'
    nav.find_element('xpath', caminho).send_keys('clima São José - SC', Keys.ENTER)
    temperatura = nav.find_element('xpath', '//*[@id="wob_tm"]').text
    vento = nav.find_element('xpath', '//*[@id="wob_ws"]').text
    umidade = nav.find_element('xpath', '//*[@id="wob_hm"]').text
    chuva = nav.find_element('xpath', '//*[@id="wob_pp"]').text
    local = nav.find_element('xpath', '//*[@id="wob_loc"]').text
    local = local.split(',')
    nav.close()

    frase = f'{usuario}, em {local[1]} A temperatura é de {temperatura} graus, ' \
            f'há {chuva} de chance de chover, o ar esta com ' \
            f'{umidade} de umidade e o vento esta soprando a {vento}.'
    return frase

print(clima('Edson'))
"""
temperatura = nav.find_element('xpath', '//*[@id="OverviewCurrentTemperature"]/a').text
temperatura = temperatura.replace("°C", "")
temperatura = temperatura.replace(" ", "")
temperatura = temperatura.replace("\n", "")
temperatura = int(temperatura)
estado = nav.find_element('xpath', '//*[@id="OverviewCurrentTemperature"]/div/div[1]').text
chuva = nav.find_element('xpath', '//*[@id="ForecastDays"]/div/ul/li[1]/button/span/'
                                  'div/div/div[2]/div[2]/div[2]/div/span[2]').text
chuva = int(chuva.replace("%", ""))
nav.quit()
frase = f'{usuario}, A temperatura é de {temperatura} graus, está {estado} e há {chuva} ' \
        f'porcento de chances de chover.'
return frase
"""


from antlr4 import *
from dctLexer import dctLexer
from dctListener import dctListener
from dctParser import dctParser
from dctVisitor import dctVisitor
import sys
from collections import defaultdict

idDict = {}
defDict = defaultdict(list)
#defDict = {}
chainDict = {}
pub = []
cert = set()

class Schema:
    pass

class Identifier:
    def __init__(self):
        self.type = None
        self.value = None
        self.signedby = None

class Expression:
    def __init__(self):
        self.type = None
        self.value = None

class CustomVisitor(dctVisitor):
    def __init__(self):
        self.definitions = []

    def visitSchema(self, ctx:dctParser.SchemaContext):
        schema = Schema()
        schema.name = "Schema"
        s = []
        for c in ctx.definition():
            s.append(c.accept(self))
        schema.definition = s
        self.definitions.append(schema)

    def visitDefinition(self, ctx:dctParser.DefinitionContext):
        id = ctx.getChild(0).accept(self)

        if(id.type == 'uString'):
            #print(type(ctx.literal().accept(self)))
            idDict[id.value] = '"' + ctx.literal().accept(self) + '"'
            #print(idDict)

        elif(id.type == 'hString'):
            exp = ctx.expression().accept(self)

            if ctx.constraints():
                defDict[id.value].append('type3')

                constraints = ctx.constraints().accept(self)

                for m in constraints:
                    for e in exp.value:
                        if(e.value in m):
                            #print(e.value)
                            e.value = m[e.value]

                defDict[id.value].append(exp.value)
                defDict[id.value].append(constraints)

            else:
                defDict[id.value].append('type5')
                defDict[id.value].append(exp.value)

        elif(id.type == 'string'):
            if not ctx.constraints():
                exp = ctx.expression().accept(self)

                if(ctx.SIGNEDBY()):
                    if(exp.type == 'id'):
                        defDict[id.value].append('type2a') # id COLON exp (id) <= cert
                    else:
                        defDict[id.value].append('type2b') # id COLON exp (name) <= cert
                    defDict[id.value].append(exp.value)
                    cert = ctx.getChild(4).accept(self)
                    defDict[id.value].append(cert)

                else:
                    defDict[id.value].append('type5') # id colon exp (name)
                    defDict[id.value].append(exp.value)

            else:
                exp = ctx.expression().accept(self)
                constraints = ctx.constraints().accept(self)

                if (not ctx.SIGNEDBY()):
                    if (exp.type == 'id'):
                        defDict[id.value].append('type3a') # id COLON exp(id) AND constraint

                    elif (exp.type == 'name'):
                        defDict[id.value].append('type3b')  # id COLON exp(name) AND constraint

                    defDict[id.value].append(exp.value)
                    defDict[id.value].append(constraints)

                else:
                    if (exp.type == 'id'):
                        defDict[id.value].append('type4a') # id COLON exp(id) AND constraint <= cert

                    elif (exp.type == 'name'):
                        defDict[id.value].append('type4b') # id COLON exp(name) AND constraint <= cert

                    defDict[id.value].append(exp.value)
                    defDict[id.value].append(constraints)

                    cert = ctx.getChild(6).accept(self)
                    #print(cert.value)
                    defDict[id.value].append(cert)

    def visitIdentifier(self, ctx:dctParser.IdentifierContext):
        id = Identifier()
        if(ctx.STRING()):
            id.type = 'string'
            id.value = ctx.STRING().getText()

        elif(ctx.ustring()):
            id.type = 'uString'
            id.value = ctx.ustring().accept(self)

        elif (ctx.hstring()):
            id.type = 'hString'
            id.value = ctx.hstring().accept(self)

        return id

    def visitConstraints(self, ctx:dctParser.ConstraintsContext):
        cl = []
        for c in ctx.constraint():
            cl.append(c.accept(self))
        return cl

    def visitConstraint(self, ctx:dctParser.ConstraintContext):
        d = {}
        for c in ctx.constraint_body():
            i, s = c.accept(self)
            d[i.value] = s
        return d

    def visitConstraint_body(self, ctx:dctParser.Constraint_bodyContext):
        id = ctx.identifier().accept(self)
        if(ctx.literal()):
            s = '"'+ ctx.literal().accept(self) + '"'
        elif(ctx.function):
            s = ctx.function().accept(self)+'()'
        return id, s

    def visitUstring(self, ctx:dctParser.UstringContext):
        return ctx.UNDERSCORE().getText() + ctx.STRING().getText()

    def visitHstring(self, ctx:dctParser.HstringContext):
        return ctx.HASH().getText() + ctx.STRING().getText()

    def visitLiteral(self, ctx:dctParser.LiteralContext):
        #print(ctx.AP().getText())
        return ctx.STRING().getText()

    def visitFunction(self, ctx:dctParser.FunctionContext):
        return ctx.STRING().getText()

    def visitExpression(self, ctx:dctParser.ExpressionContext):
        e = Expression()
        if (ctx.name()):
            e.value = ctx.name().accept(self)
            e.type = 'name'
        elif (ctx.identifier()):
            e.value = ctx.identifier().accept(self)
            e.type = 'id'
        return e

    def visitName(self, ctx:dctParser.NameContext):
        ids = []
        for c in ctx.identifier():
            ids.append(c.accept(self))
        return ids

def translate(dict,file):
    with open (file, 'w') as output:
        for key,values in dict.items():
            if values[0] in ['type2b', 'type5']:
                res = ''
                res = res + key + ' : '
                for v in values[1]:
                    if(v.type == 'uString'):
                        if v.value in idDict:
                            res += idDict[v.value] + '/'
                        else:
                            res += v.value + '/'
                    else:
                        res += v.value + '/'
                if(values[0] == 'type2b'):
                    res += ' { '+ values[2].value + ' }'
                    #print(values[2].value, key)
                    chainDict[key] = values[2].value
                    cert.add(values[2].value)
                print(res)
                output.write(res + '\n\n')

            elif values[0] == 'type2a':
                previd = dict.get(values[1].value)
                constraints = previd[2]
                for m in constraints:
                    res = ''
                    res = res + key + ' : '
                    for v in previd[1]:
                        #if(v.type == 'uString'):
                        if v.value in m:
                            res += m[v.value] + '/'
                        elif v.value in idDict:
                            res += idDict[v.value] + '/'

                        else:
                            res += v.value + '/'
                    res += ' { '+ values[2].value + ' }'
                    chainDict[key] = values[2].value
                    cert.add(values[2].value)
                    print(res)

                    output.write(res + '\n\n')

            elif values[0] in ['type3a', 'type4a']:
                previd = dict.get(values[1].value)
                if(previd != None):
                    constraints = values[2]
                    for m in constraints:
                        res = ''
                        res = res + key + ' : '
                        for v in previd[1]:
                            #if(v.type == 'uString'):
                            if v.value in m:
                                res += m[v.value] + '/'
                            elif v.value in idDict:
                                res += idDict[v.value] + '/'

                            else:
                                res += v.value + '/'
                        if(values[0] == 'type3a' and len(previd) > 2):
                            res += ' { '+ previd[2].value + ' }'
                            chainDict[key] = previd[2].value
                            cert.add(previd[2].value)
                        elif (values[0] == 'type4a'):
                            res += ' { '+ values[3].value + ' }'
                            chainDict[key] = values[3].value
                            cert.add(values[3].value)

                        print(res)
                        output.write(res + '\n\n')

            elif values[0] in ['type3b', 'type4b']:
                constraints = values[2]
                for m in constraints:
                    res = ''
                    res = res + key + ' : '
                    for v in values[1]:
                        if(v.type == 'uString'):
                            if v.value in m:
                                res += m[v.value] + '/'
                            elif v.value in idDict:
                                res += idDict[v.value] + '/'

                            else:
                                res += v.value + '/'
                        else:
                            res += v.value + '/'
                    if(values[0] == 'type4b'):
                        res += ' { '+ values[3].value + ' }'
                        chainDictdict[key] = values[3].value
                        cert.add(values[3].value)
                        print(res)
                        output.write(res + '\n\n')

def translate_signing_chain(file):
    with open (file, 'a') as output:
        output.write('\n\n\n\n')
        output.write('Signing Chain: \n\n')
        #for key, value in chainDict.items():
            #if (key in ['ocommand', 'ucommand']):
                #i = key
                #res = ''
                #while i in chainDict.keys():
                    #print(chainDict[i])
                    #res += i + ' <= '
                    #i = chainDict[i]
                #res += i
                #output.write(res + '\n\n')

        for key, value in chainDict.items():
            i = key
            res = ''
            while i in chainDict.keys():
                    #print(chainDict[i])
                res += i + ' <= '
                i = chainDict[i]
            res += i
            output.write(res + '\n\n')

def get_parse_tree(file_name):
    schema_src_code = FileStream(file_name)
    lexer = dctLexer(schema_src_code)
    stream = CommonTokenStream(lexer)
    parser = dctParser(stream)
    tree = parser.schema()
    return tree, parser.getNumberOfSyntaxErrors()

tree, err = get_parse_tree(sys.argv[1])
outputfile = sys.argv[2]
if err == 0:
    visitor = CustomVisitor()
    try:
        tree.accept(visitor)
        #print(idDict)

    except Exception as e:
        print("\nSyntax error occurred in the policy file!\n")
        sys.exit(1)

    translate(defDict, outputfile)
    print(chainDict)
    print(cert)
    translate_signing_chain(outputfile)


import random
from colorama import Fore, Back, Style

# The folowing code is from thepythoncode.com
FORES = ([Fore.BLACK, Fore.RED, Fore.GREEN, Fore.YELLOW, Fore.BLUE,
         Fore.MAGENTA, Fore.CYAN, Fore.WHITE])
BACKS = ([Back.BLACK, Back.RED, Back.GREEN, Back.YELLOW, Back.BLUE,
         Back.MAGENTA, Back.CYAN, Back.WHITE])
BRIGHTNESS = [Style.DIM, Style.NORMAL, Style.BRIGHT]

# The folowing code is from thepythoncode.com
def print_with_color(s, color=Fore.WHITE, brightness=Style.NORMAL, **kwargs):
    """Utility function wrapping the regular `print()` function
    but with colors and brightness"""
    print(f"{brightness}{color}{s}{Style.RESET_ALL}", **kwargs)

# List of words randomly generated to be guessed by user
GAME_WORDS = '''because satisfied without modest
include student grandchild anger purpose belongings commit delegate
determined powerful secret appliance application return confront
seeking daughter community experience thought future compassion
nutrition balance anybody someone spiritual various'''.split()
HANGMAN = ['''
    +----+
         |
         |
         |
        === ''', '''
    +----+
    O    |
         |
         |
        === ''', '''
    +----+
    O    |
    |    |
         |
        === ''', '''
    +----+
    O    |
   /|    |
         |
        === ''', '''
     +----+
     O    |
    /|\   |
          |
         === ''', '''
     +----+
     O    |
    /|\   |
    /     |
         === ''', '''
     +----+
     O    |
    /|\   |
    / \   |
         === ''']

def intro():
    """
    Prints a welcome message to the terminal and an input
    to get the users name. Entering non alphabetic symbols
    or an input less than 2 letters raises an error
    which is handled by a try/except statement.
    """
    print()
    print_with_color('W E L C O M E  T O  H A N G M A N\n', color=Fore.YELLOW)
    print_with_color('FIND THE WORD BEFORE YOU ARE HUNG!\n', color=Fore.YELLOW)
    print_with_color('GUESS 1 LETTER AT A TIME', color=Fore.YELLOW)
    print_with_color("OR THE FULL WORD IF YOU'RE CONFIDENT", color=Fore.YELLOW)

    while True:
        try:
            name = (str(input('\nEnter your name below:\n')))
            if name.isalpha() and len(name) > 1:
                print(f'Hello {name}, we hope you enjoy playing\n')
                print('Best of Luck!')
                break
            raise TypeError
        except TypeError:
            print('The name you entered is invalid.. Letters only')
            continue

def get_random_word(random_word):
    """
    Returns a random word from the list of game words defined above
    using the random pack imported at the top of the code.
    """
    word = random.randint(0, len(random_word) - 1)
    return random_word[word]

def game_board(wrong_guesses):
    """
    Print out the expected animation of hangman to correspond to how many
    incorrect guesses have been made.
    Also prints out an f string showing the incorrect guesses made by
    the user.
    """
    print_with_color(HANGMAN[len(wrong_guesses)], color=Fore.YELLOW)
    print()
    print_with_color(f'Guessed letters: {wrong_guesses} \n', color=Fore.YELLOW)

def hidden_answer(play_word, right_guesses):
    """
    Display dashes corresponding with the number of letters
    in the word by iterating through with a for loop.
    Updates after each user guess inserting correctly guessed
    letters obtained from the correct_guesses variable.
    """
    dashes = len(play_word) * '-'

    for i, x in enumerate(play_word):
        if play_word[i] in right_guesses:
            dashes = dashes[:i] + play_word[i] + dashes[i+1:]

    for letter in dashes:
        print(letter, end=' ')
    print()

def user_guess(guessed_letters):
    """
    Takes a users guess and establishes whether it is a
    single letter or full word guess.
    Uses a while loop and if/elif statements to ensure a valid
    guess is entered.
    """
    while True:
        print('Enter your guess below.')
        print('Either a single letter or have a go at the whole word')
        print()
        guess = input()
        guess = guess.lower()
        if len(guess) == 1:
            if guess in guessed_letters:
                print(f"\nYou've already tried {guess} ..")
                print('Pick another letter\n')
            elif guess not in 'abcdefghijklmnopqrstuvwxyz':
                print('Only guesses in the alphabet accepted.. Try again')
            else:
                return guess
        elif len(guess) == len(game_word):
            if guess == game_word:
                print_with_color('YOU GUESSED IT!', color=Fore.GREEN)
                print()
                return guess
            else:
                print_with_color('That was wrong.. Try again', color=Fore.RED)
        elif len(guess) != len(game_word):
            print("Your guess isn't the same length as the word.. ")
            print('Try again\n')
        else:
            return guess

def replay_game():
    """
    Display question to the user asking if they would like
    to play again followed by an input where either yes or no
    is typed in.
    """
    print('Would you like to play again? Yes/No\n')
    return input().lower().startswith('y')

intro()
game_word = get_random_word(GAME_WORDS)
INCORRECT_GUESSES = ''
CORRECT_GUESSES = ''
GAME_OVER = False

while True:
    # Call the game board and hidden answer functions to display
    # the animation to the user.
    game_board(INCORRECT_GUESSES)
    hidden_answer(game_word, CORRECT_GUESSES)

    # Calling user_guess function to allow the user to make a guess.
    NEW_GUESS = str(user_guess(INCORRECT_GUESSES + CORRECT_GUESSES))

    # Assign new value to correct_guesses if the users guess
    # is in the game_word.
    if NEW_GUESS in game_word:
        CORRECT_GUESSES = CORRECT_GUESSES + NEW_GUESS

        # Checking to see if player has won.
        # Iterate through game_word & compare to letters in correct_guesses.
        ALL_LETTERS_GUESSED = True
        for a, b in enumerate(game_word):
            if game_word[a] not in CORRECT_GUESSES:
                ALL_LETTERS_GUESSED = False
                break
        if ALL_LETTERS_GUESSED:
            print_with_color('CONGRATS YOU WON!!\n', color=Fore.GREEN)
            print_with_color(f'The word was {game_word}\n', color=Fore.GREEN)
            GAME_OVER = True
    else:
        INCORRECT_GUESSES = INCORRECT_GUESSES + NEW_GUESS

        # Code to execute if you've reached maximum amount of guesses.
        if len(INCORRECT_GUESSES) == len(HANGMAN) - 1:
            game_board(INCORRECT_GUESSES)
            print_with_color("AWW! YOU'RE OUT OF GUESSES..\n", color=Fore.RED)
            print_with_color("THE MAN IS HUNG!!", color=Fore.RED)
            print()
            print('The word you were searching for was', end=' ')
            print_with_color(f'{game_word}!\n', color=Fore.GREEN)
            GAME_OVER = True

    # Code to execute if the user wants to play again

    if GAME_OVER:
        if replay_game():
            game_word = get_random_word(GAME_WORDS)
            INCORRECT_GUESSES = ''
            CORRECT_GUESSES = ''
            GAME_OVER = False
        else:
            # Final question to user asking to rate game
            while True:
                try:
                    print('Before you go.. Quick question?\n')
                    FEEDBACK = int(input('What do you rate us? 1-5?\n'))
                    if FEEDBACK > 0 and FEEDBACK < 6:
                        print('Thanks for the feedback!')
                        print('Have a great day!')
                        break
                    raise ValueError
                except ValueError:
                    print('Only ratings between 1 and 5 are accepted\n')
                continue
            break


#随机抽取文件中的两个row group，计算两者之间的Jaccard作为文件的数据分布情况

import random

import numpy as np
import pyarrow.parquet as pp
from scipy.spatial.distance import pdist

def get_2_diff_index(row_group_num):
    random1 = random.random()
    random2 = random.random()
    while random1 == random2:
        random2 = random.random()
    random1 = int(random1 * row_group_num)
    random2 = int(random2 * row_group_num)
    assert random1 >= 0 and random1 < row_group_num
    assert random2 >=0 and random2 < row_group_num
    return random1, random2

def com_jaccard_np(file_path, column_name, column_type):
    _table = pp.ParquetFile(file_path)
    rg_num = _table.num_row_groups
    if rg_num < 10:
        return 0
    arr1 = np.array([])
    arr2 = np.array([])
    for rg_index in range(0, int(rg_num / 5)):
        if rg_index % 2 == 0:
            rg1_content = _table.read_row_group(rg_index, columns=[column_name])
            for _ in rg1_content.column(column_name):
                _ = str(_)
                if _ == "None":
                    continue
                if column_type == "int":
                    _ = int(_)
                elif column_type == "float":
                    _ = float(_)
                arr1 = np.append(arr1, _)
        if rg_index % 2 == 1:
            rg2_content = _table.read_row_group(rg_index, columns=[column_name])
            for _ in rg2_content.column(column_name):
                _ = str(_)
                if _ == "None":
                    continue
                if column_type == "int":
                    _ = int(_)
                elif column_type == "float":
                    _ = float(_)
                arr2 = np.append(arr2, _)
    _len = min(arr1.size, arr2.size)
    X = np.vstack([arr1[0:_len], arr2[0:_len]])
    return pdist(X, 'jaccard')

def com_jaccard_random(file_path, column_name, column_type):
    _table = pp.ParquetFile(file_path)
    rg_num = _table.num_row_groups
    if rg_num < 10:
        return 0
    union_num = 0
    join_num = 0
    rg1_dict = dict()
    _sample_num = int(rg_num/5)
    if _sample_num % 2 == 1:
        _sample_num += 1
    sample_set = set()
    while len(sample_set) < _sample_num:
        sample_set.add(random.randint(0, rg_num-1))
    cur_index = 0
    for _index in sample_set:
        if cur_index % 2 == 0:
            rg_content = _table.read_row_group(_index, columns=[column_name])
            for _ in rg_content.column(column_name):
                _ = str(_)
                if _ == "None":
                    continue
                if column_type == "int":
                    _ = int(_)
                elif column_type == "float":
                    _ = float(_)
                union_num += 1
                if _ not in rg1_dict:
                    rg1_dict[_] = 0
                rg1_dict[_] += 1
        cur_index += 1
    cur_index = 0
    for _index in sample_set:
        if cur_index % 2 == 1:
            rg_content = _table.read_row_group(_index, columns=[column_name])
            for _ in rg_content.column(column_name):
                _ = str(_)
                if _ == "None":
                    continue
                if column_type == "int":
                    _ = int(_)
                elif column_type == "float":
                    _ = float(_)
                union_num += 1
                if _ in rg1_dict:
                    join_num += (1 + rg1_dict[_])
                    rg1_dict[_] = 0
        cur_index += 1
    return join_num / union_num

def com_jaccard(file_path, column_name, column_type):
    rg1_dict = dict()
    union_num = 0
    join_num = 0
    _table = pp.ParquetFile(file_path)
    rg_num = _table.num_row_groups
    if rg_num < 10:
        return 0
    for rg_index1 in range(0, int(rg_num/5)):
        if rg_index1 % 2 == 0:
            rg1_content = _table.read_row_group(rg_index1, columns=[column_name])
            for _ in rg1_content.column(column_name):
                _ = str(_)
                if _ == "None":
                    continue
                if column_type == "int":
                    _ = int(_)
                elif column_type == "float":
                    _ = float(_)
                union_num += 1
                if _ not in rg1_dict:
                    rg1_dict[_] = 0
                rg1_dict[_] += 1
    for rg_index2 in range(0, int(rg_num/5)):
        if rg_index2 % 2 == 1:
            rg2_content = _table.read_row_group(rg_index2, columns=[column_name])
            for _ in rg2_content.column(column_name):
                _ = str(_)
                if _ == "None":
                    continue
                if column_type == "int":
                    _ = int(_)
                elif column_type == "float":
                    _ = float(_)
                union_num += 1
                if _ in rg1_dict:
                    join_num += (1 + rg1_dict[_])
                    rg1_dict[_] = 0
    return join_num/union_num

def get_file_rg_num(file_path):
    return pp.ParquetFile(file_path).num_row_groups

if __name__=="__main__":
    file_path = "/mydata/tpch_parquet_300.db_rewrite/orders/20220524_062839_00026_fs53m_66cba92c-a213-4e48-ab15-5b37a5bb0daa"
    column_name = "orderdate"
    column_type = "date"
    _jaccard = com_jaccard(file_path, column_name, column_type)
    print(file_path + "~" + column_name + "~" + str(_jaccard))  # 这个percent是1.0

    # file_path = "/mydata/tpcds_parquet_300.db_rewrite/catalog_returns/20220523_132505_00004_fs53m_509e9f99-1d58-4cf7-9d90-98845500e399"
    # column_name = "cr_item_sk"
    # column_type = "float"
    # _jaccard = com_jaccard(file_path, column_name, column_type)
    # print(file_path + "~" + column_name + "~" + str(_jaccard))#这个percent是1.0
    #
    # column_name = "cr_returned_date_sk"
    # _jaccard = com_jaccard(file_path, column_name, column_type)
    # print(file_path + "~" + column_name + "~" + str(_jaccard))  # 这个percent未知，看起来在1/4左右
    #
    # file_path = "/mydata/tpcds_parquet_300.db_rewrite/catalog_sales/20220523_132919_00005_fs53m_ff916c89-b7ca-4b23-b0f1-efd34a8ec887"
    # column_name = "cs_net_paid_inc_ship"
    # _jaccard = com_jaccard(file_path, column_name, column_type)
    # print(file_path + "~" + column_name + "~" + str(_jaccard))  # 这个percent是0.5
    #
    # file_path = "/mydata/tpch_parquet_300.db_rewrite/lineitem/20220524_040257_00019_fs53m_0ac223b1-073d-419a-a423-a83527104d5b"
    # column_name = "extendedprice"
    # _jaccard = com_jaccard(file_path, column_name, column_type)
    # print(file_path + "~" + column_name + "~" + str(_jaccard))  # 这个percent是0.75
    #
    #
    # column_name = "shipdate"
    # _jaccard = com_jaccard(file_path, column_name, "date")
    # print(file_path + "~" + column_name + "~" + str(_jaccard))  # 这个percent是0.71
    #
    # file_path = "/mydata/tpch_parquet_300.db_rewrite/customer/20220524_064704_00027_fs53m_1af48bb7-7f26-4364-8cca-751850202710"
    # column_name = "custkey"
    # _jaccard = com_jaccard(file_path, column_name, column_type)
    # print(file_path + "~" + column_name + "~" + str(_jaccard))  # 这个percent是0.004


"""The API that serves my model"""

# imports
from fastapi import FastAPI
import uvicorn
from pydantic import BaseModel, conlist
from typing import List
import joblib
from brock import CATEGORIES
import numpy as np

# load the model
model = joblib.load("model/model-final.joblib")

APP_DESC = """
##  Goals

- POC to highlight how a relatively flexible pipeline to train and serve ML models
- Flag a model that can associate one or more tags to a question with a score

### Longer term:

- all sorts of ML models can be leveraged with the tensorflow deep learning engine included
- think about real time vs batch scoring of data

### Use cases
 - Real time could include the removal of toxic questions submitted to PQCast, or suggest a set of answers to common questions.
 - In batch, tag questions and use for thought leadership and internal strategic planning (with clients) based on trends in "needs or intents"

"""

# create the app
app = FastAPI(title="POC ML API with Scikit-learn for Text Classification",
              description=APP_DESC,
              version="0.1")

# create the class for the post param
class Message(BaseModel):
    text: str
    class Config:
        schema_extra = {
        "example": {
            "text": "I like computers and internet things"
        }
    }

@app.post('/predict', tags=["predictions"])
async def get_prediction(data: Message):
    """
    For a given question text, predict if the text belongs to a target category.

     - Each question can be tagged with 1 or more categories in the taxonomy.  For example, some may have two tags.
     - Each <b>category</b> is the score of being tagged as that category in our taxonomy.  Scores go from 0 to 1, with 1 representing absolute confidence.
     - Practical usage: only associate a tag based on probability of having that tag (e.g. > .9, or some other score level)

    __Simply, ideal tag scores for a question tagged with `facts` and `admission` would be 1.0 for both of those labels.__
    """
    data = data.dict()
    question = data['text']
    pred = model.predict([question])
    int_p = int(np.round(pred, 4))
    confidence = list(model.predict_proba([question]))
    maxc = int(np.argmax(confidence[0]))
    float_c = float(np.round(confidence[0][maxc], 3))

    # return the values.  Code above removed numpy return values, which will choke FastAPI.
    resp = {'pred': int_p,
            'label': str(CATEGORIES[int_p]),
            'score': float_c}

    return resp


from django.urls import path

from .views import HomeView, EntryView, CreateEntryView, StartView

urlpatterns = [
    path('', StartView.as_view(), name = 'welcome'),
    path('list/', HomeView.as_view(), name = 'blog-home'),
    path('entry/<int:pk>/', EntryView.as_view(), name = 'entry-detail'),
    path('create_entry/', CreateEntryView.as_view(success_url='/'), name = 'entry-create')

]# -*- coding: utf-8 -*-


from datetime import date, time, datetime, timedelta

def dia_atual():
    #salva a data atual no arquivo para o usuário não tentar burlar.
    data_hj = datetime.now()
    data_str = data_hj.strftime('%d/%m/%Y')
    str_data = str(data_str)
    arquivo = open('Arquivo', 'w')
    data_arq = str_data
    arquivo.write(data_arq)
    #print(data_arq)
    arquivo.close()

print('''
Esse progrma analisa a sua qualidade de sono.
Durante 7 dias, abra esse aplicativo e coloque o horário que você se deitou para dormir, e o horário que acordou.
Será feito uma análise dos dias e horário, e será retornado a sua qualidade de sono durante os 7 dias.''')

while True:
    opcao = int(input('''
    [1] Adicionar horario.
    [2] Verificar informações cadastradas.
    [3] Sair'''))

    if opcao == 1:
        dia_atual()
        dormir = str(input('Que horas você se deitou para dormir ontem: '))
        acordar = str(input('Que horas você acordou hoje: '))
        d1 = datetime.strptime(dormir, '%H:%M')
        d2 = datetime.strptime(acordar, '%H:%M')

    elif opcao == 2:
        solicitacao = open('Arquivo', 'r')
        #str(solicitacao)
        print(solicitacao)

    elif opcao == 3:
        break


import numpy as np
import scipy.stats

class ImproperCovarianceError(Exception):
    """
    Exception to be thrown when a Gaussian is created with a covariance matrix that isn't strictly positive definite.
    """

    def __str__(self):
        return 'Covariance matrix is not strictly positive definite'

class Gaussian:
    """
    Implements a gaussian pdf. Focus is on efficient multiplication, division and sampling.
    """

    def __init__(self, m=None, P=None, U=None, S=None, Pm=None):
        """
        Initialize a gaussian pdf given a valid combination of its parameters. Valid combinations are:
        m-P, m-U, m-S, Pm-P, Pm-U, Pm-S
        :param m: mean
        :param P: precision
        :param U: upper triangular precision factor such that U'U = P
        :param S: covariance
        :param Pm: precision times mean such that P*m = Pm
        """

        try:
            if m is not None:
                m = np.asarray(m)
                self.m = m
                self.n_dims = m.size

                if P is not None:
                    P = np.asarray(P)
                    L = np.linalg.cholesky(P)
                    self.P = P
                    self.C = np.linalg.inv(L)
                    self.S = np.dot(self.C.T, self.C)
                    self.Pm = np.dot(P, m)
                    self.logdetP = 2.0 * np.sum(np.log(np.diagonal(L)))

                elif U is not None:
                    U = np.asarray(U)
                    self.P = np.dot(U.T, U)
                    self.C = np.linalg.inv(U.T)
                    self.S = np.dot(self.C.T, self.C)
                    self.Pm = np.dot(self.P, m)
                    self.logdetP = 2.0 * np.sum(np.log(np.diagonal(U)))

                elif S is not None:
                    S = np.asarray(S)
                    self.P = np.linalg.inv(S)
                    self.C = np.linalg.cholesky(S).T
                    self.S = S
                    self.Pm = np.dot(self.P, m)
                    self.logdetP = -2.0 * np.sum(np.log(np.diagonal(self.C)))

                else:
                    raise ValueError('Precision information missing.')

            elif Pm is not None:
                Pm = np.asarray(Pm)
                self.Pm = Pm
                self.n_dims = Pm.size

                if P is not None:
                    P = np.asarray(P)
                    L = np.linalg.cholesky(P)
                    self.P = P
                    self.C = np.linalg.inv(L)
                    self.S = np.dot(self.C.T, self.C)
                    self.m = np.linalg.solve(P, Pm)
                    self.logdetP = 2.0 * np.sum(np.log(np.diagonal(L)))

                elif U is not None:
                    U = np.asarray(U)
                    self.P = np.dot(U.T, U)
                    self.C = np.linalg.inv(U.T)
                    self.S = np.dot(self.C.T, self.C)
                    self.m = np.linalg.solve(self.P, Pm)
                    self.logdetP = 2.0 * np.sum(np.log(np.diagonal(U)))

                elif S is not None:
                    S = np.asarray(S)
                    self.P = np.linalg.inv(S)
                    self.C = np.linalg.cholesky(S).T
                    self.S = S
                    self.m = np.dot(S, Pm)
                    self.logdetP = -2.0 * np.sum(np.log(np.diagonal(self.C)))

                else:
                    raise ValueError('Precision information missing.')

            else:
                raise ValueError('Mean information missing.')

        except np.linalg.LinAlgError:
            raise ImproperCovarianceError()

    def gen(self, n_samples=None, rng=np.random):
        """
        Returns independent samples from the gaussian.
        """

        one_sample = n_samples is None

        z = rng.randn(1 if one_sample else n_samples, self.n_dims)
        samples = np.dot(z, self.C) + self.m

        return samples[0] if one_sample else samples

    def eval(self, x, ii=None, log=True):
        """
        Evaluates the gaussian pdf.
        :param x: rows are inputs to evaluate at
        :param ii: a list of indices specifying which marginal to evaluate. if None, the joint pdf is evaluated
        :param log: if True, the log pdf is evaluated
        :return: pdf or log pdf
        """

        x = np.asarray(x)

        if x.ndim == 1:
            return self.eval(x[np.newaxis, :], ii, log)[0]

        if ii is None:
            xm = x - self.m
            lp = -np.sum(np.dot(xm, self.P) * xm, axis=1)
            lp += self.logdetP - self.n_dims * np.log(2.0 * np.pi)
            lp *= 0.5

        else:
            m = self.m[ii]
            S = self.S[ii][:, ii]
            lp = scipy.stats.multivariate_normal.logpdf(x, m, S)
            lp = np.array([lp]) if x.shape[0] == 1 else lp

        return lp if log else np.exp(lp)

    def grad_log_p(self, x):
        """
        Evaluates the gradient of the log pdf.
        :param x: rows are inputs to evaluate at
        :return: d/dx log p(x)
        """

        x = np.asarray(x)
        return -np.dot(x - self.m, self.P)

    def __mul__(self, other):
        """
        Multiply with another gaussian.
        """

        assert isinstance(other, Gaussian)

        P = self.P + other.P
        Pm = self.Pm + other.Pm

        return Gaussian(P=P, Pm=Pm)

    def __imul__(self, other):
        """
        Incrementally multiply with another gaussian.
        """

        assert isinstance(other, Gaussian)

        res = self * other

        self.m = res.m
        self.P = res.P
        self.C = res.C
        self.S = res.S
        self.Pm = res.Pm
        self.logdetP = res.logdetP

        return res

    def __div__(self, other):
        """
        Divide by another gaussian. Note that the resulting gaussian might be improper.
        """

        assert isinstance(other, Gaussian)

        P = self.P - other.P
        Pm = self.Pm - other.Pm

        return Gaussian(P=P, Pm=Pm)

    def __idiv__(self, other):
        """
        Incrementally divide by another gaussian. Note that the resulting gaussian might be improper.
        """

        assert isinstance(other, Gaussian)

        res = self / other

        self.m = res.m
        self.P = res.P
        self.C = res.C
        self.S = res.S
        self.Pm = res.Pm
        self.logdetP = res.logdetP

        return res

    def __pow__(self, power, modulo=None):
        """
        Raise gaussian to a power and get another gaussian.
        """

        P = power * self.P
        Pm = power * self.Pm

        return Gaussian(P=P, Pm=Pm)

    def __ipow__(self, power):
        """
        Incrementally raise gaussian to a power.
        """

        res = self ** power

        self.m = res.m
        self.P = res.P
        self.C = res.C
        self.S = res.S
        self.Pm = res.Pm
        self.logdetP = res.logdetP

        return res

    def kl(self, other):
        """
        Calculates the kl divergence from this to another gaussian, i.e. KL(this | other).
        """

        assert isinstance(other, Gaussian)
        assert self.n_dims == other.n_dims

        t1 = np.sum(other.P * self.S)

        m = other.m - self.m
        t2 = np.dot(m, np.dot(other.P, m))

        t3 = self.logdetP - other.logdetP

        t = 0.5 * (t1 + t2 + t3 - self.n_dims)

        return t

def fit_gaussian(x, w=None, eps=None):
    """
    Fits and returns a gaussian to a (possibly weighted) dataset using maximum likelihood.
    :param x: data matrix, rows are datapoints
    :param w: weights for each datapoint; if None, no weighting happens
    :param eps: a constant number to add to the diagonal components of the covariance; zero if None
    """

    x = np.asarray(x)
    assert x.ndim == 2, 'wrong size'

    if w is None:

        m = np.mean(x, axis=0)
        xm = x - m
        S = np.dot(xm.T, xm) / x.shape[0]

    else:
        w = np.asarray(w)
        m = np.dot(w, x)
        S = np.dot(x.T * w, x) - np.outer(m, m)

    if eps is not None:
        S += eps * np.eye(x.shape[1])

    return Gaussian(m=m, S=S)


# -*- coding: utf-8 -*-
from os import path
from setuptools import setup

# Imports content of requirements.txt into setuptools' install_requires
with open('requirements.txt') as f:
    requirements = f.read().splitlines()

def get_version():
    with open('filesizemon.py') as f:
        for line in f:
            if line.startswith('__version__'):
                return eval(line.split('=')[-1])

# Imports content of README.md into setuptools' long_description
this_directory = path.abspath(path.dirname(__file__))
with open(path.join(this_directory, 'README.txt'), encoding='utf-8') as f:
    long_description = f.read()

setup(name='filesizemon',
      version=get_version(),
      description="Script that monitors a folder for filesize changes. Good to determine culprit during server performance analysis.",
      long_description=long_description,
      keywords="log, analysis, performance, filesize, monitor",
      author='Arturo "Buanzo" Busleiman',
      author_email='buanzo@buanzo.com.ar',
      url='https://github.com/buanzo/filesize-change-monitor',
      license='MIT License',
      zip_safe=False,
      python_requires='>=3.6',
      py_modules=['filesizemon'],
      namespace_packages=[],
      install_requires=requirements,
      entry_points={
         'console_scripts': [
            'filesizemon = filesizemon:logsizemon',
         ],
      },
      classifiers=[
         'Environment :: Console',
         'Intended Audience :: System Administrators',
         'Programming Language :: Python',
      ])


import matplotlib.pyplot as plt
import numpy as np
import random

theta = 0.639
# theta_h_data = []
# gen_risk_data = []
avg_risks = []
j = 100
ns = 10

def main():
    print(theta)
    for n in range(ns):
        total_risk = 0
        for i in range(j):
            total_risk += train(n+1)
        avg_risks.append(total_risk/j)
        print(n+1, ' : ', avg_risks[n])
    n_list = ['1','2','3','4','5','6','7','8','9','10']
    plt.bar(n_list, avg_risks)
    # plt.title('title name')
    plt.xlabel('n')
    plt.ylabel('Average Risk')
    plt.show()

def train(n):
    max_l = -1
    min_r = 1
    theta_h = -2
    for i in range(n):
        # get s_i
        x_i = random.uniform(-1,1)
        y_i = -1 if x_i<theta else 1
        # update min and max
        if y_i==-1 :
            if x_i > max_l:
                max_l = x_i
        else:
            if x_i < min_r:
                min_r = x_i
    #calc theta
    theta_h = (min_r+max_l)/2
    gen_risk = abs(theta_h-theta)/2
    # print('theta: ', theta, ', theta_h: ', theta_h, ', gen_risk: ', gen_risk)
    # theta_h_data.append(theta_h)
    # gen_risk_data.append(gen_risk)
    return gen_risk

if __name__ == "__main__":
    main()


import re
import string
import copy
import numpy as np
from nltk.corpus import stopwords
from fuzzywuzzy import process
from nltk import word_tokenize, sent_tokenize
from stanford import constituencyparser, NERtagger

def candidate_extractor(message, use_np=False):

    text = message["text"]

    stoplist = stopwords.words()
    stop = "|".join(stoplist)

    sentences = sent_tokenize(text)
    first_words = [w[0] for w in [word_tokenize(s) for s in sentences]]

    # REGEX Capitalization
    match = re.findall("(?:[{upper}]\w*(?:'s)?\s(?:(?:(?:{stop}) )*))*[{upper}]\w*".format(upper=string.ascii_uppercase, stop=stop), text)
    match = list(set(match))

    matchcopy = copy.deepcopy(match)

    for m in matchcopy:
        if 'by' in m:
            match.append(m.rsplit(' by ')[0])
            match.remove(m)

        if len(m.split()) == 1 and m in first_words:

            match.remove(m)

    # USE NP'S AS BOOKTITLES
    if use_np:
        trees = constituencyparser(text)

        if trees:
            nps = []
            for tree in trees:
                tree = list(tree)[0]
                nps += [j for j in [" ".join(i.leaves()) for i in tree[0].subtrees() if i.label() == 'NP'] if j.count(' ') > 0]

            nps_new = []
            for np in nps:
                nps_new.append(np)
                if 'by' in np:
                    nps_new.append(np.rsplit(' by ')[0])

            matches = [m for m in match if m.lower() not in stoplist and m in nps_new]
        else:
            matches = [m for m in match if m.lower() not in stoplist]
    else:
        matches = [m for m in match if m.lower() not in stoplist]

    return matches

def find_authors(text, match, windowsize=5):
    """
    Find author names (Entity Type = PERSON) within a
    windowsize (default = 5) of a matchphrase in a text.

    Uses the Stanford NERtagger to process pieces of text.
    Returns all person names as a list for every occurence in the text.
    """

    matchwords = word_tokenize(match)

    nersequence = NERtagger(text)
    tokens, etypes = zip(*nersequence)

    values = np.array(tokens)

    startindices = np.where(values == matchwords[0])
    endindices = np.where(values == matchwords[-1])

    indices = []
    for s in list(startindices)[0]:
        for e in list(endindices)[0]:
            if e-len(matchwords)+1 == s:
                start, end = int(s), int(e)
                indices.append((start,end))

    all_authors = []

    for index in set(indices):
        start, end = index

#         window = tokens[start-windowsize:start] + tokens[end+1:end+1+windowsize]
#         windowtags = etypes[start-windowsize:start] + etypes[end+1:end+1+windowsize]
#         windowrange = list(range(start-windowsize, start)) + list(range(end+1, end+1+windowsize))

        startwindow = start-windowsize
        if startwindow < 0:
            startwindow = 0

        window = tokens[startwindow:end+windowsize+1]
        windowtags = etypes[startwindow:end+windowsize+1]
        windowrange = list(range(startwindow,end+windowsize+1))

#         authors_index = [(i[0],i[2]) for i in zip(window, windowtags, windowrange) if i[1] == 'PERSON']

        authors = []
        for token, etype, index in zip(window, windowtags, windowrange):
            if etype == 'PERSON':
                authors.append(token)
            else:
                if authors:
                    all_authors.append(" ".join(authors))
                    authors = []

    return all_authors

def get_titles(c, d):

    title_candidates = set()

    for word in word_tokenize(c.lower()):
        if word in stopwords.words() or word == "'s":
            continue

        try:
            title_candidates.update(d[word])
        except:
            # print(c, "not part of a title")
            pass

    return title_candidates

def classify(message, d, threshold=90, use_np=False):
    """
    """

    ids = []
    candidates = candidate_extractor(message, use_np)

    for c in candidates:
        matchlist = get_titles(c, d)

        authors = find_authors(message["text"], c)

        # DISCARD ALL CANDIDATES THAT ARE NAMES
        if c in authors:
            continue

        authors = " ".join(authors)

        c = str(tuple((c, authors)))

        if matchlist:
            titles, authors, works = zip(*matchlist)

            # print(c)
            (title, author), confidence = process.extractOne(c, zip(titles,authors))
            workid = works[titles.index(title)]
            # print(title, authors[titles.index(title)], workid, sep=', ')

            if confidence >= threshold:
                ids.append(workid)

    if ids:
        return ids

    # if everything fails:
    return ['UNKNOWN']


# Include functions for creating plots for states that have top numbers

import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.ticker as mticker
from utilities import *

# plot accumulative graph for top10 states
def plot_accum_topN(state_data, N):
    sorted_state_data = (state_data.T).sort_values(by=['10/9/20'],
                                                   ascending=False).T
    sorted_state_data.iloc[:, :N].plot()
    # plt.show()
    try:
        plt.savefig('./plots/top_states/accum_top%d.png' % N)
    except FileNotFoundError:
        mkdir_p('./plots/top_states/')
        plt.savefig('./plots/top_states/accum_top%d.png' % N)

# plot daily graph for top5 states
def plot_daily_topN(daily_state_data, N):
    sorted_daily_state_data = (daily_state_data.T).sort_values(by=['10/9/20'],
                                                               ascending=False).T
    sorted_daily_state_data.iloc[:, :N].plot()
    # plt.show()
    try:
        plt.savefig('./plots/top_states/daily_top%d.png' % N)
    except FileNotFoundError:
        mkdir_p('./plots/top_states/')
        plt.savefig('./plots/top_states/daily_top%d.png' % N)

# plot 7-day running avg graph for top5 states
def plot_rollavg_topN(daily_state_data, N):
    roll7_state_data = daily_state_data.rolling(7).mean()
    sorted_roll7_state_data = roll7_state_data.T.sort_values(by=['10/1/20'],
                                                             ascending=False).T
    sorted_roll7_state_data.iloc[:, :N].plot()
    # plt.show()
    try:
        plt.savefig('./plots/top_states/rollavg_top%d.png' % N)
    except FileNotFoundError:
        mkdir_p('./plots/top_states/')
        plt.savefig('./plots/top_states/rollavg_top%d.png' % N)

# plot daily graph for top1 state in bar graph
def plot_rollavg_daily_topN(daily_state_data, N):
    roll7_state_data = daily_state_data.rolling(7).mean()
    sorted_roll7_state_data = roll7_state_data.T.sort_values(by=['10/1/20'],
                                                             ascending=False).T
    sorted_daily_state_data = (daily_state_data.T).sort_values(by=['10/9/20'],
                                                               ascending=False).T

    fig, ax = plt.subplots(figsize=(20, 10))
    ax = sorted_roll7_state_data.iloc[:, :N].plot(ax=ax)
    sorted_daily_state_data.iloc[:, :N].plot(kind='bar', ax=ax)
    ax.xaxis_date()
    myLocator = mticker.MultipleLocator(7)
    ax.xaxis.set_major_locator(myLocator)
    fig.autofmt_xdate()
    # plt.show()
    try:
        plt.savefig('./plots/top_states/rollavg_daily_top%d.png' % N)
    except FileNotFoundError:
        mkdir_p('./plots/top_states/')
        plt.savefig('./plots/top_states/rollavg_daily_top%d.png' % N)

if __name__ == "__main__":
    # read data
    data = pd.read_csv("./data/time_series_covid19_confirmed_US.csv", delimiter=',', index_col=0)
    temp = data.groupby(['Province_State']).sum()
    state_data = (temp.iloc[:, 5:]).T  # accumulative cases group by state
    daily_state_data = state_data.diff()  # daily cases group by state
    # print(daily_state_data.head(5))

    plot_accum_topN(state_data, 10)
    plot_daily_topN(daily_state_data, 5)
    plot_rollavg_topN(daily_state_data, 5)
    plot_rollavg_daily_topN(daily_state_data, 2)